decoder: maintain decoded frames as GstVideoCodecFrame objects.

Maintain decoded surfaces as GstVideoCodecFrame objects instead of
GstVaapiSurfaceProxy objects. The latter will tend to be reduced to
the strict minimum: a context and a surface.

gst-libs/gst/vaapi/gstvaapidecoder.c

@@ -304,29 +304,32 @@ decode_step(GstVaapiDecoder *decoder)
 }
 
 static inline void
-push_surface(GstVaapiDecoder *decoder, GstVaapiSurfaceProxy *proxy)
+push_frame(GstVaapiDecoder *decoder, GstVideoCodecFrame *frame)
 {
     GstVaapiDecoderPrivate * const priv = decoder->priv;
-    GstVideoInfo * const vi = &priv->codec_state->info;
-    GstClockTime duration;
 
     GST_DEBUG("queue decoded surface %" GST_VAAPI_ID_FORMAT,
-              GST_VAAPI_ID_ARGS(gst_vaapi_surface_proxy_get_surface_id(proxy)));
+              GST_VAAPI_ID_ARGS(gst_vaapi_surface_proxy_get_surface_id(
+                                    frame->user_data)));
 
-    if (vi->fps_n && vi->fps_d) {
+    g_queue_push_tail(priv->frames, frame);
-        /* Actual field duration is computed in vaapipostproc */
-        duration = gst_util_uint64_scale(GST_SECOND, vi->fps_d, vi->fps_n);
-        gst_vaapi_surface_proxy_set_duration(proxy, duration);
-    }
-    g_queue_push_tail(priv->surfaces, proxy);
 }
 
-static inline GstVaapiSurfaceProxy *
+static inline GstVideoCodecFrame *
-pop_surface(GstVaapiDecoder *decoder)
+pop_frame(GstVaapiDecoder *decoder)
 {
     GstVaapiDecoderPrivate * const priv = decoder->priv;
+    GstVideoCodecFrame *frame;
 
-    return g_queue_pop_head(priv->surfaces);
+    frame = g_queue_pop_head(priv->frames);
+    if (!frame)
+        return NULL;
+
+    GST_DEBUG("dequeue decoded surface %" GST_VAAPI_ID_FORMAT,
+              GST_VAAPI_ID_ARGS(gst_vaapi_surface_proxy_get_surface_id(
+                                    frame->user_data)));
+
+    return frame;
 }
 
 static void
@@ -387,11 +390,11 @@ gst_vaapi_decoder_finalize(GObject *object)
         priv->buffers = NULL;
     }
 
-    if (priv->surfaces) {
+    if (priv->frames) {
-        clear_queue(priv->surfaces, (GDestroyNotify)
+        clear_queue(priv->frames, (GDestroyNotify)
-            gst_vaapi_surface_proxy_unref);
+            gst_video_codec_frame_unref);
-        g_queue_free(priv->surfaces);
+        g_queue_free(priv->frames);
-        priv->surfaces = NULL;
+        priv->frames = NULL;
     }
 
     g_clear_object(&priv->context);
@@ -507,7 +510,7 @@ gst_vaapi_decoder_init(GstVaapiDecoder *decoder)
     priv->codec                 = 0;
     priv->codec_state           = codec_state;
     priv->buffers               = g_queue_new();
-    priv->surfaces              = g_queue_new();
+    priv->frames                = g_queue_new();
 }
 
 /**
@@ -605,7 +608,7 @@ GstVaapiDecoderStatus
 gst_vaapi_decoder_get_surface(GstVaapiDecoder *decoder,
     GstVaapiSurfaceProxy **out_proxy_ptr)
 {
-    GstVaapiSurfaceProxy *proxy;
+    GstVideoCodecFrame *frame;
     GstVaapiDecoderStatus status;
 
     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder),
@@ -613,20 +616,24 @@ gst_vaapi_decoder_get_surface(GstVaapiDecoder *decoder,
     g_return_val_if_fail(out_proxy_ptr != NULL,
         GST_VAAPI_DECODER_STATUS_ERROR_INVALID_PARAMETER);
 
-    proxy = pop_surface(decoder);
+    frame = pop_frame(decoder);
-    if (!proxy) {
+    if (!frame) {
         do {
             status = decode_step(decoder);
         } while (status == GST_VAAPI_DECODER_STATUS_SUCCESS);
-        proxy = pop_surface(decoder);
+        frame = pop_frame(decoder);
     }
 
-    if (proxy)
+    if (frame) {
+        *out_proxy_ptr = gst_vaapi_surface_proxy_ref(frame->user_data);
+        gst_video_codec_frame_unref(frame);
         status = GST_VAAPI_DECODER_STATUS_SUCCESS;
-    else if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
+    }
-        status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
+    else {
-
+        *out_proxy_ptr = NULL;
-    *out_proxy_ptr = proxy;
+        if (status == GST_VAAPI_DECODER_STATUS_SUCCESS)
+            status = GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
+    }
     return status;
 }
 
@@ -635,9 +642,11 @@ gst_vaapi_decoder_get_surface(GstVaapiDecoder *decoder,
  * @decoder: a #GstVaapiDecoder
  * @out_frame_ptr: the next decoded frame as a #GstVideoCodecFrame
  *
- * Returns the next frame available in the list of decoded frames.
+ * On successful return, *@out_frame_ptr contains the next decoded
- * @GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA is returned if there is no
+ * frame available as a #GstVideoCodecFrame. The caller owns this
- * decoded frame pending in the queue.
+ * object, so gst_video_codec_frame_unref() shall be called after
+ * usage. Otherwise, @GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA is
+ * returned if no decoded frame is available.
  *
  * The actual surface is available as a #GstVaapiSurfaceProxy attached
  * to the user-data anchor of the output frame. Ownership of the proxy
@@ -649,7 +658,6 @@ GstVaapiDecoderStatus
 gst_vaapi_decoder_get_frame(GstVaapiDecoder *decoder,
     GstVideoCodecFrame **out_frame_ptr)
 {
-    GstVaapiSurfaceProxy *proxy;
     GstVideoCodecFrame *out_frame;
 
     g_return_val_if_fail(GST_VAAPI_IS_DECODER(decoder),
@@ -657,20 +665,10 @@ gst_vaapi_decoder_get_frame(GstVaapiDecoder *decoder,
     g_return_val_if_fail(out_frame_ptr != NULL,
         GST_VAAPI_DECODER_STATUS_ERROR_INVALID_PARAMETER);
 
-    proxy = pop_surface(decoder);
+    out_frame = pop_frame(decoder);
-    if (!proxy || !(out_frame = gst_vaapi_surface_proxy_get_user_data(proxy)))
+    if (!out_frame)
         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 
-    gst_video_codec_frame_set_user_data(out_frame,
-        proxy, (GDestroyNotify)gst_vaapi_mini_object_unref);
-
-    out_frame->pts      = GST_VAAPI_SURFACE_PROXY_TIMESTAMP(proxy);
-    out_frame->duration = GST_VAAPI_SURFACE_PROXY_DURATION(proxy);
-
-    if (GST_VAAPI_SURFACE_PROXY_TFF(proxy))
-        GST_VIDEO_CODEC_FRAME_FLAG_SET(out_frame,
-            GST_VIDEO_CODEC_FRAME_FLAG_TFF);
-
     *out_frame_ptr = out_frame;
     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 }
@@ -809,12 +807,10 @@ gst_vaapi_decoder_ensure_context(
 }
 
 void
-gst_vaapi_decoder_push_surface_proxy(
+gst_vaapi_decoder_push_frame(GstVaapiDecoder *decoder,
-    GstVaapiDecoder      *decoder,
+    GstVideoCodecFrame *frame)
-    GstVaapiSurfaceProxy *proxy
-)
 {
-    push_surface(decoder, proxy);
+    push_frame(decoder, frame);
 }
 
 GstVaapiDecoderStatus

gst-libs/gst/vaapi/gstvaapidecoder_objects.c

@@ -84,6 +84,11 @@ gst_vaapi_picture_destroy(GstVaapiPicture *picture)
 
     vaapi_destroy_buffer(GET_VA_DISPLAY(picture), &picture->param_id);
     picture->param = NULL;
+
+    if (picture->frame) {
+        gst_video_codec_frame_unref(picture->frame);
+        picture->frame = NULL;
+    }
 }
 
 gboolean
@@ -164,9 +169,8 @@ gst_vaapi_picture_create(
     if (!picture->slices)
         return FALSE;
 
-    gst_vaapi_mini_object_set_user_data(
+    picture->frame = gst_video_codec_frame_ref(
-        GST_VAAPI_MINI_OBJECT(picture->proxy),
+        GST_VAAPI_DECODER_CODEC_FRAME(GET_DECODER(picture)));
-        GST_VAAPI_DECODER_CODEC_FRAME(GET_DECODER(picture)), NULL);
     return TRUE;
 }
 
@@ -304,21 +308,40 @@ gboolean
 gst_vaapi_picture_output(GstVaapiPicture *picture)
 {
     GstVaapiSurfaceProxy *proxy;
+    GstVideoCodecFrame *out_frame;
 
     g_return_val_if_fail(GST_VAAPI_IS_PICTURE(picture), FALSE);
 
     if (!picture->proxy)
         return FALSE;
 
-    if (!GST_VAAPI_PICTURE_IS_SKIPPED(picture)) {
+    out_frame = gst_video_codec_frame_ref(picture->frame);
-        proxy = gst_vaapi_surface_proxy_ref(picture->proxy);
+
-        gst_vaapi_surface_proxy_set_timestamp(proxy, picture->pts);
+    proxy = gst_vaapi_surface_proxy_ref(picture->proxy);
-        if (GST_VAAPI_PICTURE_IS_INTERLACED(picture))
+    gst_video_codec_frame_set_user_data(out_frame,
-            gst_vaapi_surface_proxy_set_interlaced(proxy, TRUE);
+        proxy, (GDestroyNotify)gst_vaapi_mini_object_unref);
-        if (GST_VAAPI_PICTURE_IS_TFF(picture))
+
-            gst_vaapi_surface_proxy_set_tff(proxy, TRUE);
+    if (!GST_CLOCK_TIME_IS_VALID(out_frame->pts))
-        gst_vaapi_decoder_push_surface_proxy(GET_DECODER(picture), proxy);
+        out_frame->pts = picture->pts;
-    }
+    if (GST_VAAPI_PICTURE_IS_SKIPPED(picture))
+        GST_VIDEO_CODEC_FRAME_FLAG_SET(out_frame,
+            GST_VIDEO_CODEC_FRAME_FLAG_DECODE_ONLY);
+    if (GST_VAAPI_PICTURE_IS_TFF(picture))
+        GST_VIDEO_CODEC_FRAME_FLAG_SET(out_frame,
+            GST_VIDEO_CODEC_FRAME_FLAG_TFF);
+
+    /* XXX: to be removed later */
+    if (GST_CLOCK_TIME_IS_VALID(out_frame->pts))
+        gst_vaapi_surface_proxy_set_timestamp(proxy, out_frame->pts);
+    if (GST_CLOCK_TIME_IS_VALID(out_frame->duration))
+        gst_vaapi_surface_proxy_set_duration(proxy, out_frame->duration);
+    if (GST_VAAPI_PICTURE_IS_INTERLACED(picture))
+        gst_vaapi_surface_proxy_set_interlaced(proxy, TRUE);
+    if (GST_VAAPI_PICTURE_IS_TFF(picture))
+        gst_vaapi_surface_proxy_set_tff(proxy, TRUE);
+
+    gst_vaapi_decoder_push_frame(GET_DECODER(picture), out_frame);
+
     GST_VAAPI_PICTURE_FLAG_SET(picture, GST_VAAPI_PICTURE_FLAG_OUTPUT);
     return TRUE;
 }

gst-libs/gst/vaapi/gstvaapidecoder_objects.h

@@ -114,6 +114,7 @@ enum {
 struct _GstVaapiPicture {
     /*< private >*/
     GstVaapiCodecObject         parent_instance;
+    GstVideoCodecFrame         *frame;
     GstVaapiSurface            *surface;
     GstVaapiSurfaceProxy       *proxy;
     VABufferID                  param_id;

gst-libs/gst/vaapi/gstvaapidecoder_priv.h

@@ -163,7 +163,7 @@ struct _GstVaapiDecoderPrivate {
     GstVaapiCodec       codec;
     GstVideoCodecState *codec_state;
     GQueue             *buffers;
-    GQueue             *surfaces;
+    GQueue             *frames;
     GstVaapiParserState parser_state;
 };
 
@@ -209,10 +209,8 @@ gst_vaapi_decoder_ensure_context(
 
 G_GNUC_INTERNAL
 void
-gst_vaapi_decoder_push_surface_proxy(
+gst_vaapi_decoder_push_frame(GstVaapiDecoder *decoder,
-    GstVaapiDecoder      *decoder,
+    GstVideoCodecFrame *frame);
-    GstVaapiSurfaceProxy *proxy
-);
 
 G_GNUC_INTERNAL
 GstVaapiDecoderStatus

gst/vaapi/gstvaapidecode.c

@@ -234,6 +234,7 @@ gst_vaapidecode_handle_frame(GstVideoDecoder *vdec, GstVideoCodecFrame *frame)
         ret = gst_video_decoder_finish_frame(vdec, out_frame);
         if (ret != GST_FLOW_OK)
             goto error_commit_buffer;
+        gst_video_codec_frame_unref(out_frame);
     };
     return GST_FLOW_OK;
 
@@ -269,12 +270,14 @@ error_create_buffer:
                   "surface %" GST_VAAPI_ID_FORMAT,
                   GST_VAAPI_ID_ARGS(surface_id));
         gst_video_decoder_drop_frame(vdec, out_frame);
+        gst_video_codec_frame_unref(out_frame);
         return GST_FLOW_UNEXPECTED;
     }
 error_commit_buffer:
     {
         GST_DEBUG("video sink rejected the video buffer (error %d)", ret);
         gst_video_decoder_drop_frame(vdec, out_frame);
+        gst_video_codec_frame_unref(out_frame);
         return GST_FLOW_UNEXPECTED;
     }
 }