va: Add and use gst_va_base_dec_prepare_output_frame().

This helper will do downstream negotiation and later will
allocate the output frame.

H265 and AV1 decoders don't use this approach since their output
frame allocation is different.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3480>

subprojects/gst-plugins-bad/sys/va/gstvabasedec.c

@@ -1052,3 +1052,21 @@ gst_va_base_dec_process_output (GstVaBaseDec * base, GstVideoCodecFrame * frame,
 
   return TRUE;
 }
+
+GstFlowReturn
+gst_va_base_dec_prepare_output_frame (GstVaBaseDec * base,
+    GstVideoCodecFrame * frame)
+{
+  GstVideoDecoder *vdec = GST_VIDEO_DECODER (base);
+
+  if (base->need_negotiation) {
+    if (!gst_video_decoder_negotiate (vdec)) {
+      GST_ERROR_OBJECT (base, "Failed to negotiate with downstream");
+      return GST_FLOW_NOT_NEGOTIATED;
+    }
+  }
+
+  if (frame)
+    return gst_video_decoder_allocate_output_frame (vdec, frame);
+  return GST_FLOW_OK;
+}

subprojects/gst-plugins-bad/sys/va/gstvabasedec.h

@@ -135,4 +135,7 @@ gboolean              gst_va_base_dec_copy_output_buffer  (GstVaBaseDec * base,
 gboolean              gst_va_base_dec_process_output      (GstVaBaseDec * base,
                                                            GstVideoCodecFrame * frame,
                                                            GstVideoBufferFlags buffer_flags);
+GstFlowReturn         gst_va_base_dec_prepare_output_frame (GstVaBaseDec * base,
+                                                            GstVideoCodecFrame * frame);
+
 G_END_DECLS

subprojects/gst-plugins-bad/sys/va/gstvah264dec.c

@@ -474,19 +474,11 @@ gst_va_h264_dec_new_picture (GstH264Decoder * decoder,
     GstVideoCodecFrame * frame, GstH264Picture * picture)
 {
   GstVaH264Dec *self = GST_VA_H264_DEC (decoder);
-  GstVaDecodePicture *pic;
-  GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
   GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
+  GstVaDecodePicture *pic;
   GstFlowReturn ret;
 
-  if (base->need_negotiation) {
+  ret = gst_va_base_dec_prepare_output_frame (base, frame);
-    if (!gst_video_decoder_negotiate (vdec)) {
-      GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
-      return GST_FLOW_NOT_NEGOTIATED;
-    }
-  }
-
-  ret = gst_video_decoder_allocate_output_frame (vdec, frame);
   if (ret != GST_FLOW_OK)
     goto error;
 

subprojects/gst-plugins-bad/sys/va/gstvajpegdec.c

@@ -139,7 +139,6 @@ gst_va_jpeg_dec_new_picture (GstJpegDecoder * decoder,
     GstJpegFrameHdr * frame_hdr)
 {
   GstVaJpegDec *self = GST_VA_JPEG_DEC (decoder);
-  GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
   GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
   GstFlowReturn ret;
   VAProfile profile;
@@ -171,14 +170,7 @@ gst_va_jpeg_dec_new_picture (GstJpegDecoder * decoder,
         gst_va_profile_name (profile), rt_format, base->width, base->height);
   }
 
-  if (base->need_negotiation) {
+  ret = gst_va_base_dec_prepare_output_frame (base, frame);
-    if (!gst_video_decoder_negotiate (vdec)) {
-      GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
-      return GST_FLOW_NOT_NEGOTIATED;
-    }
-  }
-
-  ret = gst_video_decoder_allocate_output_frame (vdec, frame);
   if (ret != GST_FLOW_OK) {
     GST_ERROR_OBJECT (self, "Failed to allocate output buffer: %s",
         gst_flow_get_name (ret));

subprojects/gst-plugins-bad/sys/va/gstvampeg2dec.c

@@ -289,20 +289,12 @@ static GstFlowReturn
 gst_va_mpeg2_dec_new_picture (GstMpeg2Decoder * decoder,
     GstVideoCodecFrame * frame, GstMpeg2Picture * picture)
 {
-  GstFlowReturn ret;
   GstVaMpeg2Dec *self = GST_VA_MPEG2_DEC (decoder);
-  GstVaDecodePicture *pic;
   GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
-  GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
+  GstVaDecodePicture *pic;
+  GstFlowReturn ret;
 
-  if (base->need_negotiation) {
+  ret = gst_va_base_dec_prepare_output_frame (base, frame);
-    if (!gst_video_decoder_negotiate (vdec)) {
-      GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
-      return GST_FLOW_NOT_NEGOTIATED;
-    }
-  }
-
-  ret = gst_video_decoder_allocate_output_frame (vdec, frame);
   if (ret != GST_FLOW_OK)
     goto error;
 

subprojects/gst-plugins-bad/sys/va/gstvavp8dec.c

@@ -188,18 +188,10 @@ gst_va_vp8_dec_new_picture (GstVp8Decoder * decoder,
 {
   GstVaVp8Dec *self = GST_VA_VP8_DEC (decoder);
   GstVaDecodePicture *pic;
-  GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
   GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
   GstFlowReturn ret;
 
-  if (base->need_negotiation) {
+  ret = gst_va_base_dec_prepare_output_frame (base, frame);
-    if (!gst_video_decoder_negotiate (vdec)) {
-      GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
-      return GST_FLOW_NOT_NEGOTIATED;
-    }
-  }
-
-  ret = gst_video_decoder_allocate_output_frame (vdec, frame);
   if (ret != GST_FLOW_OK)
     goto error;
 

subprojects/gst-plugins-bad/sys/va/gstvavp9dec.c

@@ -216,21 +216,13 @@ gst_va_vp9_dec_new_picture (GstVp9Decoder * decoder,
   GstFlowReturn ret;
   GstVaVp9Dec *self = GST_VA_VP9_DEC (decoder);
   GstVaDecodePicture *pic;
-  GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
   GstVaBaseDec *base = GST_VA_BASE_DEC (decoder);
 
   ret = _check_resolution_change (self, picture);
   if (ret != GST_FLOW_OK)
     return ret;
 
-  if (base->need_negotiation) {
+  ret = gst_va_base_dec_prepare_output_frame (base, frame);
-    if (!gst_video_decoder_negotiate (vdec)) {
-      GST_ERROR_OBJECT (self, "Failed to negotiate with downstream");
-      return GST_FLOW_NOT_NEGOTIATED;
-    }
-  }
-
-  ret = gst_video_decoder_allocate_output_frame (vdec, frame);
   if (ret != GST_FLOW_OK)
     goto error;