codecs: av1: Fix state when we show existing keyframe

Showing existing keyframe have special meaning in AV1. All the references
frame will be refreshed with the original keyframe information. The refresh
process (7.20) is implemented by saving data from the frame_header into the
state. To fix this special case, load all the relevant information into the
frame_header.

As there is nothing happening in between this and the loading of the key-frame
into the state, this patch also remove the separate API function, using it
internally instead.

Fixes #1090

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1971>

subprojects/gst-docs/symbols/symbol_index.json

@@ -37478,7 +37478,6 @@
   "gst_av1_parser_parse_temporal_delimiter_obu",
   "gst_av1_parser_parse_tile_group_obu",
   "gst_av1_parser_parse_tile_list_obu",
-  "gst_av1_parser_reference_frame_loading",
   "gst_av1_parser_reference_frame_update",
   "gst_av1_parser_reset",
   "gst_av1_parser_reset_annex_b",
@@ -66199,4 +66198,4 @@
   "zbar:message",
   "zebrastripe",
   "zebrastripe:threshold"
-]
+]

subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gstav1parser.c

@@ -67,10 +67,6 @@
  * should call gst_av1_parser_reference_frame_update() to update the parser's inside
  * state(such as reference information, global segmentation information, etc).
  *
- * Note: If the frame is actived by show_existing_frame in #GST_AV1_OBU_FRAME_HEADER,
- * the function of gst_av1_parser_reference_frame_loading() should be called before
- * really showing that frame.
- *
  * @since: 1.18.00
  */
 
@@ -3496,6 +3492,64 @@ gst_av1_set_frame_refs (GstAV1Parser * parser,
       frame_header->ref_frame_idx[i] = ref;
 }
 
+/* 7.21 */
+static void
+gst_av1_parser_reference_frame_loading (GstAV1Parser * parser,
+    GstAV1FrameHeaderOBU * frame_header)
+{
+  GstAV1ReferenceFrameInfo *ref_info = &(parser->state.ref_info);
+  gint idx = frame_header->frame_to_show_map_idx;
+  GstAV1TileInfo *ref_tile_info = &ref_info->entry[idx].ref_tile_info;
+  const gint all_frames = (1 << GST_AV1_NUM_REF_FRAMES) - 1;
+
+  /* copy the relevant frame information as these will be needed by
+   * all subclasses. */
+  frame_header->frame_type = ref_info->entry[idx].ref_frame_type;
+  frame_header->upscaled_width = ref_info->entry[idx].ref_upscaled_width;
+  frame_header->frame_width = ref_info->entry[idx].ref_frame_width;
+  frame_header->frame_height = ref_info->entry[idx].ref_frame_height;
+  frame_header->render_width = ref_info->entry[idx].ref_render_width;
+  frame_header->render_height = ref_info->entry[idx].ref_render_height;
+
+  if (parser->seq_header->film_grain_params_present)
+    frame_header->film_grain_params =
+        ref_info->entry[idx].ref_film_grain_params;
+
+  /* the remaining is only relevant to ensure proper state update and only
+   * keyframe updates the state. */
+  if (frame_header->frame_type != GST_AV1_KEY_FRAME)
+    return;
+
+  frame_header->refresh_frame_flags = all_frames;
+  frame_header->current_frame_id = ref_info->entry[idx].ref_frame_id;
+  frame_header->order_hint = ref_info->entry[idx].ref_order_hint;
+  frame_header->segmentation_params =
+      ref_info->entry[idx].ref_segmentation_params;
+  frame_header->global_motion_params =
+      ref_info->entry[idx].ref_global_motion_params;
+  frame_header->loop_filter_params = ref_info->entry[idx].ref_lf_params;
+  frame_header->tile_info = *ref_tile_info;
+
+  parser->state.current_frame_id = ref_info->entry[idx].ref_frame_id;
+  parser->state.upscaled_width = ref_info->entry[idx].ref_upscaled_width;
+  parser->state.frame_width = ref_info->entry[idx].ref_frame_width;
+  parser->state.frame_height = ref_info->entry[idx].ref_frame_height;
+  parser->state.render_width = ref_info->entry[idx].ref_render_width;
+  parser->state.render_height = ref_info->entry[idx].ref_render_height;
+  parser->state.mi_cols = ref_info->entry[idx].ref_mi_cols;
+  parser->state.mi_rows = ref_info->entry[idx].ref_mi_rows;
+
+  memcpy (parser->state.mi_col_starts, ref_tile_info->mi_col_starts,
+      sizeof (guint32) * (GST_AV1_MAX_TILE_COLS + 1));
+  memcpy (parser->state.mi_row_starts, ref_tile_info->mi_row_starts,
+      sizeof (guint32) * (GST_AV1_MAX_TILE_ROWS + 1));
+  parser->state.tile_cols_log2 = ref_tile_info->tile_cols_log2;
+  parser->state.tile_cols = ref_tile_info->tile_cols;
+  parser->state.tile_rows_log2 = ref_tile_info->tile_rows_log2;
+  parser->state.tile_rows = ref_tile_info->tile_rows;
+  parser->state.tile_size_bytes = ref_tile_info->tile_size_bytes;
+}
+
 /* 5.9.2 */
 static GstAV1ParserResult
 gst_av1_parse_uncompressed_frame_header (GstAV1Parser * parser, GstAV1OBU * obu,
@@ -3581,16 +3635,7 @@ gst_av1_parse_uncompressed_frame_header (GstAV1Parser * parser, GstAV1OBU * obu,
         }
       }
 
-      frame_header->frame_type =
-          ref_info->entry[frame_header->frame_to_show_map_idx].ref_frame_type;
-      if (frame_header->frame_type == GST_AV1_KEY_FRAME) {
-        frame_header->refresh_frame_flags = all_frames;
-      }
-
-      /* just use the frame_to_show's grain_params
-       * if (seq_header->film_grain_params_present)
-       *     load_grain_params () */
-
+      gst_av1_parser_reference_frame_loading (parser, frame_header);
       goto success;
     }
 
@@ -4174,74 +4219,6 @@ error:
   return retval;
 }
 
-/* 7.21 */
-/**
- * gst_av1_parser_reference_frame_loading:
- * @parser: the #GstAV1Parser
- * @frame_header: a #GstAV1FrameHeaderOBU to load
- *
- * Load the context of @frame_header to parser's state. This function is
- * used when we want to show already parsed frames before.
- *
- * Returns: The #GstAV1ParserResult.
- *
- * Since: 1.18
- */
-GstAV1ParserResult
-gst_av1_parser_reference_frame_loading (GstAV1Parser * parser,
-    GstAV1FrameHeaderOBU * frame_header)
-{
-  GstAV1ReferenceFrameInfo *ref_info;
-  GstAV1TileInfo *ref_tile_info;
-
-  g_return_val_if_fail (parser != NULL, GST_AV1_PARSER_INVALID_OPERATION);
-  g_return_val_if_fail (frame_header != NULL, GST_AV1_PARSER_INVALID_OPERATION);
-
-  if (!parser->seq_header) {
-    GST_WARNING ("Missing OBU Reference: seq_header");
-    return GST_AV1_PARSER_MISSING_OBU_REFERENCE;
-  }
-
-  ref_info = &(parser->state.ref_info);
-
-  if (frame_header->frame_to_show_map_idx > GST_AV1_NUM_REF_FRAMES - 1)
-    return GST_AV1_PARSER_BITSTREAM_ERROR;
-
-  g_assert (ref_info->entry[frame_header->frame_to_show_map_idx].ref_valid);
-
-  parser->state.current_frame_id =
-      ref_info->entry[frame_header->frame_to_show_map_idx].ref_frame_id;
-  parser->state.upscaled_width =
-      ref_info->entry[frame_header->frame_to_show_map_idx].ref_upscaled_width;
-  parser->state.frame_width =
-      ref_info->entry[frame_header->frame_to_show_map_idx].ref_frame_width;
-  parser->state.frame_height =
-      ref_info->entry[frame_header->frame_to_show_map_idx].ref_frame_height;
-  parser->state.render_width =
-      ref_info->entry[frame_header->frame_to_show_map_idx].ref_render_width;
-  parser->state.render_height =
-      ref_info->entry[frame_header->frame_to_show_map_idx].ref_render_height;
-  parser->state.mi_cols =
-      ref_info->entry[frame_header->frame_to_show_map_idx].ref_mi_cols;
-  parser->state.mi_rows =
-      ref_info->entry[frame_header->frame_to_show_map_idx].ref_mi_rows;
-
-  ref_tile_info =
-      &ref_info->entry[frame_header->frame_to_show_map_idx].ref_tile_info;
-
-  memcpy (parser->state.mi_col_starts, ref_tile_info->mi_col_starts,
-      sizeof (guint32) * (GST_AV1_MAX_TILE_COLS + 1));
-  memcpy (parser->state.mi_row_starts, ref_tile_info->mi_row_starts,
-      sizeof (guint32) * (GST_AV1_MAX_TILE_ROWS + 1));
-  parser->state.tile_cols_log2 = ref_tile_info->tile_cols_log2;
-  parser->state.tile_cols = ref_tile_info->tile_cols;
-  parser->state.tile_rows_log2 = ref_tile_info->tile_rows_log2;
-  parser->state.tile_rows = ref_tile_info->tile_rows;
-  parser->state.tile_size_bytes = ref_tile_info->tile_size_bytes;
-
-  return GST_AV1_PARSER_OK;
-}
-
 /**
  * gst_av1_parser_reference_frame_update:
  * @parser: the #GstAV1Parser

subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gstav1parser.h

@@ -1820,11 +1820,6 @@ GstAV1ParserResult
 gst_av1_parser_parse_frame_obu (GstAV1Parser * parser, GstAV1OBU * obu,
     GstAV1FrameOBU * frame);
 
-GST_CODEC_PARSERS_API
-GstAV1ParserResult
-gst_av1_parser_reference_frame_loading (GstAV1Parser * parser,
-    GstAV1FrameHeaderOBU * frame_header);
-
 GST_CODEC_PARSERS_API
 GstAV1ParserResult
 gst_av1_parser_reference_frame_update (GstAV1Parser * parser,

subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstav1decoder.c

@@ -363,12 +363,6 @@ gst_av1_decoder_decode_frame_header (GstAV1Decoder * self,
       return GST_FLOW_ERROR;
     }
 
-    if (gst_av1_parser_reference_frame_loading (priv->parser,
-            &ref_picture->frame_hdr) != GST_AV1_PARSER_OK) {
-      GST_WARNING_OBJECT (self, "load the reference frame failed");
-      return GST_FLOW_ERROR;
-    }
-
     /* FIXME: duplicate picture might be optional feature like that of VP9
      * decoder baseclass */
     g_assert (klass->duplicate_picture);
@@ -380,8 +374,6 @@ gst_av1_decoder_decode_frame_header (GstAV1Decoder * self,
 
     picture->system_frame_number = priv->current_frame->system_frame_number;
     picture->frame_hdr = *frame_header;
-    picture->frame_hdr.render_width = ref_picture->frame_hdr.render_width;
-    picture->frame_hdr.render_height = ref_picture->frame_hdr.render_height;
     priv->current_picture = picture;
   } else {
     picture = gst_av1_picture_new ();

subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/gstvaapidecoder_av1.c

@@ -753,12 +753,6 @@ av1_decode_frame_header (GstVaapiDecoderAV1 * decoder,
       return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
     }
 
-    if (gst_av1_parser_reference_frame_loading (priv->parser,
-            &to_show_picture->frame_header) != GST_AV1_PARSER_OK) {
-      GST_ERROR ("load frame to show ref frame failed");
-      return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
-    }
-
     picture = (GstVaapiPictureAV1 *)
         gst_vaapi_picture_new_clone (GST_VAAPI_PICTURE_CAST (to_show_picture));
     if (!picture)
@@ -769,15 +763,6 @@ av1_decode_frame_header (GstVaapiDecoderAV1 * decoder,
     GST_VAAPI_PICTURE_FLAG_UNSET (picture, GST_VAAPI_PICTURE_FLAG_SKIPPED);
 
     picture->frame_header = to_show_picture->frame_header;
-    /* only update references if the frame_to_show_map_idx is a KEY FRAME */
-    if (picture->frame_header.frame_type == GST_AV1_KEY_FRAME) {
-      picture->frame_header = to_show_picture->frame_header;
-      g_assert (picture->frame_header.refresh_frame_flags ==
-          ((1 << GST_AV1_NUM_REF_FRAMES) - 1));
-    } else {
-      /* Just set to no update ref */
-      picture->frame_header.refresh_frame_flags = 0;
-    }
   } else {
     /* Resolution changed */
     if (priv->width != priv->seq_header->max_frame_width_minus_1 + 1 ||