va: encoder: extend prepare_output() virtual function

The output of VP9 and AV1 encoder is a little different from the H264
and H265 encoder, it may contain repeat frames and so the output frame
number may be more than the input. We need to call finish_subframe()
when some frame will be repeated later. So we need to extend the
current prepare_output() virtual function.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3015>

subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c

@@ -269,7 +269,7 @@ gst_va_base_enc_import_input_buffer (GstVaBaseEnc * base,
   return gst_va_buffer_importer_import (&importer, inbuf, buf);
 }
 
-static GstBuffer *
+GstBuffer *
 gst_va_base_enc_create_output_buffer (GstVaBaseEnc * base,
     GstVaEncodePicture * picture)
 {
@@ -404,37 +404,38 @@ config_failed:
 static GstFlowReturn
 _push_buffer_to_downstream (GstVaBaseEnc * base, GstVideoCodecFrame * frame)
 {
-  GstVaEncodePicture *enc_picture;
   GstVaBaseEncClass *base_class = GST_VA_BASE_ENC_GET_CLASS (base);
-  GstBuffer *buf;
+  GstFlowReturn ret;
+  gboolean complete = TRUE;
 
-  if (base_class->prepare_output)
+  if (!base_class->prepare_output (base, frame, &complete)) {
-    base_class->prepare_output (base, frame);
+    GST_ERROR_OBJECT (base, "Failed to prepare output");
-
-  enc_picture =
-      *((GstVaEncodePicture **) gst_video_codec_frame_get_user_data (frame));
-
-  buf = gst_va_base_enc_create_output_buffer (base, enc_picture);
-  if (!buf) {
-    GST_ERROR_OBJECT (base, "Failed to create output buffer");
     goto error;
   }
 
-  gst_buffer_replace (&frame->output_buffer, buf);
+  if (frame->output_buffer)
-  gst_clear_buffer (&buf);
+    GST_LOG_OBJECT (base, "Push to downstream: frame system_frame_number: %d,"
+        " pts: %" GST_TIME_FORMAT ", dts: %" GST_TIME_FORMAT
+        " duration: %" GST_TIME_FORMAT ", buffer size: %" G_GSIZE_FORMAT,
+        frame->system_frame_number, GST_TIME_ARGS (frame->pts),
+        GST_TIME_ARGS (frame->dts), GST_TIME_ARGS (frame->duration),
+        gst_buffer_get_size (frame->output_buffer));
 
-  GST_LOG_OBJECT (base, "Push to downstream: frame system_frame_number: %d,"
+  if (complete) {
-      " pts: %" GST_TIME_FORMAT ", dts: %" GST_TIME_FORMAT
+    ret = gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (base), frame);
-      " duration: %" GST_TIME_FORMAT ", buffer size: %" G_GSIZE_FORMAT,
+  } else {
-      frame->system_frame_number, GST_TIME_ARGS (frame->pts),
+    if (frame->output_buffer) {
-      GST_TIME_ARGS (frame->dts), GST_TIME_ARGS (frame->duration),
+      ret = gst_video_encoder_finish_subframe (GST_VIDEO_ENCODER (base), frame);
-      gst_buffer_get_size (frame->output_buffer));
+    } else {
+      /* Allow to output later and no data here. */
+      ret = GST_FLOW_OK;
+    }
+  }
 
-  return gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (base), frame);
+  return ret;
 
 error:
   gst_clear_buffer (&frame->output_buffer);
-  gst_clear_buffer (&buf);
   gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (base), frame);
 
   return GST_FLOW_ERROR;

subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h

@@ -91,8 +91,9 @@ struct _GstVaBaseEncClass
   GstFlowReturn (*encode_frame) (GstVaBaseEnc * encoder,
                                  GstVideoCodecFrame * frame,
                                  gboolean is_last);
-  void     (*prepare_output) (GstVaBaseEnc * encoder,
+  gboolean (*prepare_output) (GstVaBaseEnc * encoder,
-                              GstVideoCodecFrame * frame);
+                              GstVideoCodecFrame * frame,
+                              gboolean * complete);
 
   GstVaCodecs codec;
   VAEntrypoint entrypoint;
@@ -136,6 +137,8 @@ gboolean              gst_va_base_enc_add_trellis_parameter (GstVaBaseEnc * base
 void                  gst_va_base_enc_add_codec_tag       (GstVaBaseEnc * base,
                                                            const gchar * codec_name);
 void                  gst_va_base_enc_reset_state         (GstVaBaseEnc * base);
+GstBuffer *           gst_va_base_enc_create_output_buffer (GstVaBaseEnc * base,
+                                                            GstVaEncodePicture * picture);
 
 void                  gst_va_base_enc_update_property_uint (GstVaBaseEnc * base,
                                                             guint32 * old_val,

subprojects/gst-plugins-bad/sys/va/gstvah264enc.c

@@ -3038,11 +3038,13 @@ gst_va_h264_enc_flush (GstVideoEncoder * venc)
   return GST_VIDEO_ENCODER_CLASS (parent_class)->flush (venc);
 }
 
-static void
+static gboolean
-gst_va_h264_enc_prepare_output (GstVaBaseEnc * base, GstVideoCodecFrame * frame)
+gst_va_h264_enc_prepare_output (GstVaBaseEnc * base,
+    GstVideoCodecFrame * frame, gboolean * complete)
 {
   GstVaH264Enc *self = GST_VA_H264_ENC (base);
   GstVaH264EncFrame *frame_enc;
+  GstBuffer *buf;
 
   frame_enc = _enc_frame (frame);
 
@@ -3054,6 +3056,18 @@ gst_va_h264_enc_prepare_output (GstVaBaseEnc * base, GstVideoCodecFrame * frame)
       (gint64) self->gop.num_reorder_frames);
   base->output_frame_count++;
   frame->duration = base->frame_duration;
+
+  buf = gst_va_base_enc_create_output_buffer (base, frame_enc->picture);
+  if (!buf) {
+    GST_ERROR_OBJECT (base, "Failed to create output buffer");
+    return FALSE;
+  }
+
+  gst_buffer_replace (&frame->output_buffer, buf);
+  gst_clear_buffer (&buf);
+
+  *complete = TRUE;
+  return TRUE;
 }
 
 static gint

subprojects/gst-plugins-bad/sys/va/gstvah265enc.c

@@ -4611,11 +4611,13 @@ gst_va_h265_enc_new_frame (GstVaBaseEnc * base, GstVideoCodecFrame * frame)
   return TRUE;
 }
 
-static void
+static gboolean
-gst_va_h265_enc_prepare_output (GstVaBaseEnc * base, GstVideoCodecFrame * frame)
+gst_va_h265_enc_prepare_output (GstVaBaseEnc * base,
+    GstVideoCodecFrame * frame, gboolean * complete)
 {
   GstVaH265Enc *self = GST_VA_H265_ENC (base);
   GstVaH265EncFrame *frame_enc;
+  GstBuffer *buf;
 
   frame_enc = _enc_frame (frame);
 
@@ -4627,6 +4629,18 @@ gst_va_h265_enc_prepare_output (GstVaBaseEnc * base, GstVideoCodecFrame * frame)
       (gint64) self->gop.num_reorder_frames);
   base->output_frame_count++;
   frame->duration = base->frame_duration;
+
+  buf = gst_va_base_enc_create_output_buffer (base, frame_enc->picture);
+  if (!buf) {
+    GST_ERROR_OBJECT (base, "Failed to create output buffer");
+    return FALSE;
+  }
+
+  gst_buffer_replace (&frame->output_buffer, buf);
+  gst_clear_buffer (&buf);
+
+  *complete = TRUE;
+  return TRUE;
 }
 
 /* *INDENT-OFF* */