codecs: Use GST_VIDEO_DECODER_ERROR() only for decoding error case

The GST_VIDEO_DECODER_ERROR() should be used only for robust/error-resilient
decoding purpose. Any other error codes such as not-negotiated or flushing
should be returned without modified for upstream to be able to handle
it immediately. (for example, application might want to try other
decoder element on not-negotiated)

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1070>

subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c

@@ -647,13 +647,11 @@ out:
   priv->current_picture = NULL;
   priv->current_frame = NULL;
 
-  if (ret != GST_FLOW_OK) {
+  if (ret == GST_FLOW_ERROR) {
     GST_VIDEO_DECODER_ERROR (decoder, 1, STREAM, DECODE,
         ("Failed to handle the frame %d", frame->system_frame_number),
         NULL, ret);
-
-    return ret;
   }
 
-  return GST_FLOW_OK;
+  return ret;
 }

subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth264decoder.c

@@ -563,10 +563,12 @@ gst_h264_decoder_handle_frame (GstVideoDecoder * decoder,
   gst_buffer_unmap (in_buf, &map);
 
   if (decode_ret != GST_FLOW_OK) {
+    if (decode_ret == GST_FLOW_ERROR) {
       GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE,
           ("Failed to decode data"), (NULL), decode_ret);
-    gst_video_decoder_drop_frame (decoder, frame);
+    }
 
+    gst_video_decoder_drop_frame (decoder, frame);
     gst_h264_picture_clear (&priv->current_picture);
     priv->current_frame = NULL;
 
@@ -577,7 +579,7 @@ gst_h264_decoder_handle_frame (GstVideoDecoder * decoder,
   gst_video_codec_frame_unref (frame);
   priv->current_frame = NULL;
 
-  if (decode_ret != GST_FLOW_OK) {
+  if (decode_ret == GST_FLOW_ERROR) {
     GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE,
         ("Failed to decode data"), (NULL), decode_ret);
   }

subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.c

@@ -1804,10 +1804,12 @@ gst_h265_decoder_handle_frame (GstVideoDecoder * decoder,
   priv->current_frame = NULL;
 
   if (decode_ret != GST_FLOW_OK) {
+    if (decode_ret == GST_FLOW_ERROR) {
       GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE,
           ("Failed to decode data"), (NULL), decode_ret);
-    gst_video_decoder_drop_frame (decoder, frame);
+    }
 
+    gst_video_decoder_drop_frame (decoder, frame);
     gst_h265_picture_clear (&priv->current_picture);
 
     return decode_ret;
@@ -1821,7 +1823,7 @@ gst_h265_decoder_handle_frame (GstVideoDecoder * decoder,
     gst_video_decoder_release_frame (decoder, frame);
   }
 
-  if (decode_ret != GST_FLOW_OK) {
+  if (decode_ret == GST_FLOW_ERROR) {
     GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE,
         ("Failed to decode data"), (NULL), decode_ret);
   }

subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstmpeg2decoder.c

@@ -1184,13 +1184,17 @@ gst_mpeg2_decoder_handle_frame (GstVideoDecoder * decoder,
 
 failed:
   {
+    if (ret == GST_FLOW_ERROR) {
       GST_VIDEO_DECODER_ERROR (decoder, 1, STREAM, DECODE,
           ("failed to handle the frame %d", frame->system_frame_number), (NULL),
           ret);
+    }
+
     gst_video_decoder_drop_frame (decoder, frame);
     gst_mpeg2_picture_clear (&priv->current_picture);
     gst_mpeg2_picture_clear (&priv->first_field);
     priv->current_frame = NULL;
+
     return ret;
   }
 }

subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstvp8decoder.c

@@ -362,6 +362,7 @@ gst_vp8_decoder_handle_frame (GstVideoDecoder * decoder,
 
   if (!gst_buffer_map (in_buf, &map, GST_MAP_READ)) {
     GST_ERROR_OBJECT (self, "Cannot map buffer");
+    ret = GST_FLOW_ERROR;
     goto error;
   }
 
@@ -370,6 +371,7 @@ gst_vp8_decoder_handle_frame (GstVideoDecoder * decoder,
 
   if (pres != GST_VP8_PARSER_OK) {
     GST_ERROR_OBJECT (self, "Cannot parser frame header");
+    ret = GST_FLOW_ERROR;
     goto unmap_and_error;
   }
 
@@ -473,12 +475,12 @@ error:
     if (picture)
       gst_vp8_picture_unref (picture);
 
-    if (ret == GST_FLOW_OK)
+    if (ret == GST_FLOW_ERROR) {
-      ret = GST_FLOW_ERROR;
-
-    gst_video_decoder_drop_frame (decoder, frame);
       GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE,
           ("Failed to decode data"), (NULL), ret);
+    }
+
+    gst_video_decoder_drop_frame (decoder, frame);
 
     return ret;
   }

subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstvp9decoder.c

@@ -335,6 +335,7 @@ gst_vp9_decoder_handle_frame (GstVideoDecoder * decoder,
 
   if (!gst_buffer_map (in_buf, &map, GST_MAP_READ)) {
     GST_ERROR_OBJECT (self, "Cannot map input buffer");
+    ret = GST_FLOW_ERROR;
     goto error;
   }
 
@@ -343,6 +344,7 @@ gst_vp9_decoder_handle_frame (GstVideoDecoder * decoder,
 
   if (pres != GST_VP9_PARSER_OK) {
     GST_ERROR_OBJECT (self, "Failed to parsing frame header");
+    ret = GST_FLOW_ERROR;
     goto unmap_and_error;
   }
 
@@ -498,12 +500,12 @@ error:
     if (picture)
       gst_vp9_picture_unref (picture);
 
-    if (ret == GST_FLOW_OK)
+    if (ret == GST_FLOW_ERROR) {
-      ret = GST_FLOW_ERROR;
-
-    gst_video_decoder_drop_frame (decoder, frame);
       GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE,
           ("Failed to decode data"), (NULL), ret);
+    }
+
+    gst_video_decoder_drop_frame (decoder, frame);
 
     return ret;
   }