vdpau: fixup GstFlowReturn handling

Previously the different decoders would discard errounous GstFlowReturns coming
from downstream. Now we properly return these further upstream so that we
properly error out on eg. negotiation problems.

sys/vdpau/basevideodecoder/gstbasevideodecoder.c

@@ -499,20 +499,20 @@ gst_base_video_decoder_sink_query (GstPad * pad, GstQuery * query)
   return res;
 }
 
-void
+gboolean
 gst_base_video_decoder_set_src_caps (GstBaseVideoDecoder * base_video_decoder)
 {
   GstCaps *caps;
   GstVideoState *state = &base_video_decoder->state;
 
   if (base_video_decoder->have_src_caps)
-    return;
+    return TRUE;
 
   caps = gst_pad_get_allowed_caps (base_video_decoder->srcpad);
   if (!caps)
-    goto null_caps;
+    goto null_allowed_caps;
   if (gst_caps_is_empty (caps))
-    goto empty_caps;
+    goto empty_allowed_caps;
 
   gst_caps_set_simple (caps,
       "width", G_TYPE_INT, state->width,
@@ -526,25 +526,26 @@ gst_base_video_decoder_set_src_caps (GstBaseVideoDecoder * base_video_decoder)
 
 
   gst_pad_fixate_caps (base_video_decoder->srcpad, caps);
-
-
   GST_DEBUG ("setting caps %" GST_PTR_FORMAT, caps);
 
-  gst_pad_set_caps (GST_BASE_VIDEO_DECODER_SRC_PAD (base_video_decoder), caps);
+  base_video_decoder->have_src_caps =
+      gst_pad_set_caps (GST_BASE_VIDEO_DECODER_SRC_PAD (base_video_decoder),
+      caps);
+  gst_caps_unref (caps);
 
-  base_video_decoder->have_src_caps = TRUE;
+  return base_video_decoder->have_src_caps;
+
+null_allowed_caps:
+  GST_ERROR_OBJECT (base_video_decoder,
+      "Got null from gst_pad_get_allowed_caps");
+  return FALSE;
+
+empty_allowed_caps:
+  GST_ERROR_OBJECT (base_video_decoder,
+      "Got EMPTY caps from gst_pad_get_allowed_caps");
 
   gst_caps_unref (caps);
-  return;
+  return FALSE;
-
-null_caps:
-  GST_WARNING ("Got null caps from get_allowed_caps");
-  return;
-
-empty_caps:
-  GST_WARNING ("Got empty caps from get_allowed_caps");
-  gst_caps_unref (caps);
-  return;
 }
 
 static GstFlowReturn
@@ -870,6 +871,10 @@ gst_base_video_decoder_finish_frame (GstBaseVideoDecoder * base_video_decoder,
   base_video_decoder_class =
       GST_BASE_VIDEO_DECODER_GET_CLASS (base_video_decoder);
 
+
+  if (!gst_base_video_decoder_set_src_caps (base_video_decoder))
+    return GST_FLOW_NOT_NEGOTIATED;
+
   gst_base_video_decoder_calculate_timestamps (base_video_decoder, frame,
       &presentation_timestamp, &presentation_duration);
 
@@ -913,8 +918,6 @@ gst_base_video_decoder_finish_frame (GstBaseVideoDecoder * base_video_decoder,
   GST_DEBUG ("pushing frame %" GST_TIME_FORMAT,
       GST_TIME_ARGS (presentation_timestamp));
 
-  gst_base_video_decoder_set_src_caps (base_video_decoder);
-
   if (base_video_decoder->sink_clipping) {
     gint64 start = GST_BUFFER_TIMESTAMP (src_buffer);
     gint64 stop = GST_BUFFER_TIMESTAMP (src_buffer) +
@@ -1050,7 +1053,6 @@ gst_base_video_decoder_set_state (GstBaseVideoDecoder * base_video_decoder,
   base_video_decoder->state = state;
 
   base_video_decoder->have_src_caps = FALSE;
-  gst_base_video_decoder_set_src_caps (base_video_decoder);
 }
 
 void

sys/vdpau/basevideodecoder/gstbasevideodecoder.h

@@ -169,7 +169,7 @@ gst_base_video_decoder_have_frame (GstBaseVideoDecoder *base_video_decoder,
 GstVideoState gst_base_video_decoder_get_state (GstBaseVideoDecoder *base_video_decoder);
 void gst_base_video_decoder_set_state (GstBaseVideoDecoder *base_video_decoder,
     GstVideoState state);
-void gst_base_video_decoder_set_src_caps (GstBaseVideoDecoder * base_video_decoder);
+gboolean gst_base_video_decoder_set_src_caps (GstBaseVideoDecoder * base_video_decoder);
 
 void gst_base_video_decoder_lost_sync (GstBaseVideoDecoder *base_video_decoder);
 

sys/vdpau/gstvdp/gstvdpdecoder.c

@@ -154,6 +154,10 @@ gst_vdp_decoder_init_decoder (GstVdpDecoder * vdp_decoder,
       goto destroy_decoder_error;
   }
 
+  if (!gst_base_video_decoder_set_src_caps (GST_BASE_VIDEO_DECODER
+          (vdp_decoder)))
+    return GST_FLOW_NOT_NEGOTIATED;
+
   state =
       gst_base_video_decoder_get_state (GST_BASE_VIDEO_DECODER (vdp_decoder));
 

sys/vdpau/gstvdpvideopostprocess.c

@@ -526,11 +526,9 @@ gst_vdp_vpp_sink_setcaps (GstPad * pad, GstCaps * caps)
 
   allowed_caps = gst_pad_get_allowed_caps (vpp->srcpad);
   if (G_UNLIKELY (!allowed_caps))
-    goto allowed_caps_error;
+    goto null_allowed_caps;
-  if (G_UNLIKELY (gst_caps_is_empty (allowed_caps))) {
+  if (G_UNLIKELY (gst_caps_is_empty (allowed_caps)))
-    gst_caps_unref (allowed_caps);
+    goto empty_allowed_caps;
-    goto allowed_caps_error;
-  }
   GST_DEBUG ("allowed_caps: %" GST_PTR_FORMAT, allowed_caps);
 
   output_caps = gst_vdp_video_to_output_caps (video_caps);
@@ -569,8 +567,14 @@ done:
 
   return res;
 
-allowed_caps_error:
+null_allowed_caps:
-  GST_ERROR_OBJECT (vpp, "Got invalid allowed caps");
+  GST_ERROR_OBJECT (vpp, "Got null from gst_pad_get_allowed_caps");
+  goto done;
+
+empty_allowed_caps:
+  GST_ERROR_OBJECT (vpp, "Got EMPTY caps from gst_pad_get_allowed_caps");
+
+  gst_caps_unref (allowed_caps);
   goto done;
 
 not_negotiated:

sys/vdpau/h264/gsth264dpb.c

@@ -81,21 +81,24 @@ gst_h264_dpb_remove (GstH264DPB * dpb, guint idx)
     frames[i] = frames[i + 1];
 }
 
-static void
+static GstFlowReturn
 gst_h264_dpb_output (GstH264DPB * dpb, guint idx)
 {
+  GstFlowReturn ret;
   GstH264Frame *frame = dpb->frames[idx];
 
   gst_video_frame_ref (GST_VIDEO_FRAME_CAST (frame));
-  dpb->output (dpb, frame, dpb->user_data);
+  ret = dpb->output (dpb, frame, dpb->user_data);
   frame->output_needed = FALSE;
 
   if (!frame->is_reference)
     gst_h264_dpb_remove (dpb, idx);
+
+  return ret;
 }
 
 static gboolean
-gst_h264_dpb_bump (GstH264DPB * dpb, guint poc)
+gst_h264_dpb_bump (GstH264DPB * dpb, guint poc, GstFlowReturn * ret)
 {
   GstH264Frame **frames;
   guint i;
@@ -118,7 +121,7 @@ gst_h264_dpb_bump (GstH264DPB * dpb, guint poc)
     }
 
     if (frames[bump_idx]->poc < poc) {
-      gst_h264_dpb_output (dpb, bump_idx);
+      *ret = gst_h264_dpb_output (dpb, bump_idx);
       return TRUE;
     }
   }
@@ -126,10 +129,11 @@ gst_h264_dpb_bump (GstH264DPB * dpb, guint poc)
   return FALSE;
 }
 
-gboolean
+GstFlowReturn
 gst_h264_dpb_add (GstH264DPB * dpb, GstH264Frame * h264_frame)
 {
   GstH264Frame **frames;
+  GstFlowReturn ret;
 
   GST_DEBUG ("add frame with poc: %d", h264_frame->poc);
 
@@ -140,33 +144,40 @@ gst_h264_dpb_add (GstH264DPB * dpb, GstH264Frame * h264_frame)
     h264_frame->is_reference = FALSE;
 
   if (h264_frame->is_reference) {
+
+    ret = GST_FLOW_OK;
     while (dpb->n_frames == dpb->max_frames) {
-      if (!gst_h264_dpb_bump (dpb, G_MAXUINT)) {
+      if (!gst_h264_dpb_bump (dpb, G_MAXUINT, &ret)) {
         GST_ERROR_OBJECT (dpb, "Couldn't make room in DPB");
-        return FALSE;
+        return GST_FLOW_OK;
       }
     }
     dpb->frames[dpb->n_frames++] = h264_frame;
   }
 
   else {
-    while (gst_h264_dpb_bump (dpb, h264_frame->poc));
+    while (gst_h264_dpb_bump (dpb, h264_frame->poc, &ret)) {
-    dpb->output (dpb, h264_frame, dpb->user_data);
+      if (ret != GST_FLOW_OK)
+        return ret;
+    }
+
+    ret = dpb->output (dpb, h264_frame, dpb->user_data);
   }
 
-  return TRUE;
+  return ret;
 }
 
 void
 gst_h264_dpb_flush (GstH264DPB * dpb, gboolean output)
 {
+  GstFlowReturn ret;
   GstVideoFrame **frames;
   guint i;
 
   GST_DEBUG ("flush");
 
   if (output)
-    while (gst_h264_dpb_bump (dpb, G_MAXUINT));
+    while (gst_h264_dpb_bump (dpb, G_MAXUINT, &ret));
 
   frames = (GstVideoFrame **) dpb->frames;
   for (i = 0; i < dpb->n_frames; i++)
@@ -334,11 +345,12 @@ gst_h264_dpb_set_property (GObject * object, guint property_id,
   switch (property_id) {
     case PROP_NUM_REF_FRAMES:
     {
+      GstFlowReturn ret;
       guint i;
 
       dpb->max_frames = g_value_get_uint (value);
       for (i = dpb->n_frames; i > dpb->max_frames; i--)
-        gst_h264_dpb_bump (dpb, G_MAXUINT);
+        gst_h264_dpb_bump (dpb, G_MAXUINT, &ret);
 
       break;
     }

sys/vdpau/h264/gsth264dpb.h

@@ -42,7 +42,7 @@ G_BEGIN_DECLS
 typedef struct _GstH264DPB GstH264DPB;
 typedef struct _GstH264DPBClass GstH264DPBClass;
 
-typedef void (*GstH264DPBOutputFunc) (GstH264DPB *dpb, GstH264Frame *h264_frame, gpointer user_data);
+typedef GstFlowReturn (*GstH264DPBOutputFunc) (GstH264DPB *dpb, GstH264Frame *h264_frame, gpointer user_data);
 
 struct _GstH264DPB
 {

sys/vdpau/h264/gstvdph264dec.c

@@ -151,7 +151,7 @@ gst_vdp_h264_dec_set_sink_caps (GstBaseVideoDecoder * base_video_decoder,
   return TRUE;
 }
 
-static void
+static GstFlowReturn
 gst_vdp_h264_dec_output (GstH264DPB * dpb, GstH264Frame * h264_frame,
     gpointer user_data)
 {
@@ -159,7 +159,7 @@ gst_vdp_h264_dec_output (GstH264DPB * dpb, GstH264Frame * h264_frame,
 
   GST_DEBUG ("poc: %d", h264_frame->poc);
 
-  gst_base_video_decoder_finish_frame (base_video_decoder,
+  return gst_base_video_decoder_finish_frame (base_video_decoder,
       GST_VIDEO_FRAME_CAST (h264_frame));
 }
 
@@ -168,7 +168,8 @@ gst_vdp_h264_dec_calculate_poc (GstVdpH264Dec * h264_dec, GstH264Slice * slice)
 {
   GstH264Picture *pic;
   GstH264Sequence *seq;
-  guint poc = 0;
+
+  guint poc;
 
   pic = slice->picture;
   seq = pic->sequence;
@@ -259,7 +260,7 @@ gst_vdp_h264_dec_calculate_par (GstH264VUIParameters * vui, guint16 * par_n,
   return FALSE;
 }
 
-static gboolean
+static GstFlowReturn
 gst_vdp_h264_dec_idr (GstVdpH264Dec * h264_dec, GstH264Frame * h264_frame)
 {
   GstH264Slice *slice;
@@ -496,7 +497,7 @@ gst_vdp_h264_dec_handle_frame (GstBaseVideoDecoder * base_video_decoder,
       h264_dec->got_idr = TRUE;
     else {
       gst_base_video_decoder_skip_frame (base_video_decoder, frame);
-      return ret;
+      return GST_FLOW_OK;
     }
   }
 
@@ -585,9 +586,7 @@ gst_vdp_h264_dec_handle_frame (GstBaseVideoDecoder * base_video_decoder,
       gst_h264_dpb_mark_sliding (h264_dec->dpb);
   }
 
-  gst_h264_dpb_add (h264_dec->dpb, h264_frame);
+  return gst_h264_dpb_add (h264_dec->dpb, h264_frame);
-
-  return GST_FLOW_OK;
 }
 
 static gint

sys/vdpau/mpeg/gstvdpmpegdec.c

@@ -290,7 +290,7 @@ gst_vdp_mpeg_dec_handle_frame (GstBaseVideoDecoder * base_video_decoder,
   VdpPictureInfoMPEG1Or2 *info;
   GstVdpMpegFrame *mpeg_frame;
 
-  GstFlowReturn ret;
+  GstFlowReturn ret = GST_FLOW_OK;
   VdpBitstreamBuffer vbit[1];
   GstVdpVideoBuffer *outbuf;
 
@@ -356,7 +356,7 @@ gst_vdp_mpeg_dec_handle_frame (GstBaseVideoDecoder * base_video_decoder,
 
   if (info->picture_coding_type != B_FRAME) {
     if (info->backward_reference != VDP_INVALID_HANDLE) {
-      gst_base_video_decoder_finish_frame (base_video_decoder,
+      ret = gst_base_video_decoder_finish_frame (base_video_decoder,
           mpeg_dec->b_frame);
     }
 
@@ -371,6 +371,11 @@ gst_vdp_mpeg_dec_handle_frame (GstBaseVideoDecoder * base_video_decoder,
     info->backward_reference = VDP_INVALID_HANDLE;
   }
 
+  if (ret != GST_FLOW_OK) {
+    gst_base_video_decoder_skip_frame (base_video_decoder, frame);
+    return ret;
+  }
+
   /* decode */
   vbit[0].struct_version = VDP_BITSTREAM_BUFFER_VERSION;
   vbit[0].bitstream = GST_BUFFER_DATA (mpeg_frame->slices);

sys/vdpau/mpeg4/gstvdpmpeg4dec.c

@@ -244,7 +244,7 @@ gst_vdp_mpeg4_dec_handle_frame (GstBaseVideoDecoder * base_video_decoder,
   if (vop.coding_type != B_VOP) {
     if (mpeg4_dec->b_frame) {
 
-      gst_base_video_decoder_finish_frame (base_video_decoder,
+      ret = gst_base_video_decoder_finish_frame (base_video_decoder,
           GST_VIDEO_FRAME_CAST (mpeg4_dec->b_frame));
 
       if (mpeg4_dec->f_frame)