nvcodec: nvh264sldec: Add support for interlaced stream

Implement missing interlaced stream support

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1026>

subprojects/gst-plugins-bad/sys/nvcodec/gstnvdecoder.c

@@ -1472,8 +1472,8 @@ gst_nv_decoder_negotiate (GstNvDecoder * decoder,
   }
 
   info = &decoder->info;
-  state = gst_video_decoder_set_output_state (videodec,
+  state = gst_video_decoder_set_interlaced_output_state (videodec,
-      GST_VIDEO_INFO_FORMAT (info),
+      GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_INTERLACE_MODE (info),
       GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info), input_state);
   state->caps = gst_video_info_to_caps (&state->info);
 

subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264dec.c

@@ -89,9 +89,6 @@ struct _GstNvH264Dec
 
   GstVideoCodecState *output_state;
 
-  const GstH264SPS *last_sps;
-  const GstH264PPS *last_pps;
-
   GstCudaContext *context;
   GstNvDecoder *decoder;
   CUVIDPICPARAMS params;
@@ -112,6 +109,10 @@ struct _GstNvH264Dec
   guint bitdepth;
   guint chroma_format_idc;
   gint max_dpb_size;
+
+  gboolean interlaced;
+
+  GArray *ref_list;
 };
 
 struct _GstNvH264DecClass
@@ -123,6 +124,7 @@ struct _GstNvH264DecClass
 #define gst_nv_h264_dec_parent_class parent_class
 G_DEFINE_TYPE (GstNvH264Dec, gst_nv_h264_dec, GST_TYPE_H264_DECODER);
 
+static void gst_nv_h264_decoder_dispose (GObject * object);
 static void gst_nv_h264_decoder_finalize (GObject * object);
 static void gst_nv_h264_dec_set_context (GstElement * element,
     GstContext * context);
@@ -139,6 +141,8 @@ static gboolean gst_nv_h264_dec_new_sequence (GstH264Decoder * decoder,
     const GstH264SPS * sps, gint max_dpb_size);
 static gboolean gst_nv_h264_dec_new_picture (GstH264Decoder * decoder,
     GstVideoCodecFrame * frame, GstH264Picture * picture);
+static gboolean gst_nv_h264_dec_new_field_picture (GstH264Decoder *
+    decoder, const GstH264Picture * first_field, GstH264Picture * second_field);
 static GstFlowReturn gst_nv_h264_dec_output_picture (GstH264Decoder *
     decoder, GstVideoCodecFrame * frame, GstH264Picture * picture);
 static gboolean gst_nv_h264_dec_start_picture (GstH264Decoder * decoder,
@@ -166,6 +170,7 @@ gst_nv_h264_dec_class_init (GstNvH264DecClass * klass)
    * Since: 1.18
    */
 
+  object_class->dispose = gst_nv_h264_decoder_dispose;
   object_class->finalize = gst_nv_h264_decoder_finalize;
 
   element_class->set_context = GST_DEBUG_FUNCPTR (gst_nv_h264_dec_set_context);
@@ -181,6 +186,8 @@ gst_nv_h264_dec_class_init (GstNvH264DecClass * klass)
       GST_DEBUG_FUNCPTR (gst_nv_h264_dec_new_sequence);
   h264decoder_class->new_picture =
       GST_DEBUG_FUNCPTR (gst_nv_h264_dec_new_picture);
+  h264decoder_class->new_field_picture =
+      GST_DEBUG_FUNCPTR (gst_nv_h264_dec_new_field_picture);
   h264decoder_class->output_picture =
       GST_DEBUG_FUNCPTR (gst_nv_h264_dec_output_picture);
   h264decoder_class->start_picture =
@@ -201,6 +208,20 @@ gst_nv_h264_dec_class_init (GstNvH264DecClass * klass)
 static void
 gst_nv_h264_dec_init (GstNvH264Dec * self)
 {
+  self->ref_list = g_array_sized_new (FALSE, TRUE,
+      sizeof (GstH264Picture *), 16);
+  g_array_set_clear_func (self->ref_list,
+      (GDestroyNotify) gst_h264_picture_clear);
+}
+
+static void
+gst_nv_h264_decoder_dispose (GObject * object)
+{
+  GstNvH264Dec *self = GST_NV_H264_DEC (object);
+
+  g_clear_pointer (&self->ref_list, g_array_unref);
+
+  G_OBJECT_CLASS (parent_class)->dispose (object);
 }
 
 static void
@@ -246,6 +267,7 @@ gst_d3d11_h264_dec_reset (GstNvH264Dec * self)
   self->bitdepth = 0;
   self->chroma_format_idc = 0;
   self->max_dpb_size = 0;
+  self->interlaced = FALSE;
 }
 
 static gboolean
@@ -344,6 +366,7 @@ gst_nv_h264_dec_new_sequence (GstH264Decoder * decoder, const GstH264SPS * sps,
   GstNvH264Dec *self = GST_NV_H264_DEC (decoder);
   gint crop_width, crop_height;
   gboolean modified = FALSE;
+  gboolean interlaced;
 
   GST_LOG_OBJECT (self, "new sequence");
 
@@ -378,6 +401,13 @@ gst_nv_h264_dec_new_sequence (GstH264Decoder * decoder, const GstH264SPS * sps,
     modified = TRUE;
   }
 
+  interlaced = !sps->frame_mbs_only_flag;
+  if (self->interlaced != interlaced) {
+    GST_INFO_OBJECT (self, "interlaced sequence changed");
+    self->interlaced = interlaced;
+    modified = TRUE;
+  }
+
   if (self->max_dpb_size < max_dpb_size) {
     GST_INFO_OBJECT (self, "Requires larger DPB size (%d -> %d)",
         self->max_dpb_size, max_dpb_size);
@@ -408,6 +438,8 @@ gst_nv_h264_dec_new_sequence (GstH264Decoder * decoder, const GstH264SPS * sps,
     }
 
     gst_video_info_set_format (&info, out_format, self->width, self->height);
+    if (self->interlaced)
+      GST_VIDEO_INFO_INTERLACE_MODE (&info) = GST_VIDEO_INTERLACE_MODE_MIXED;
 
     self->max_dpb_size = max_dpb_size;
     /* FIXME: add support cudaVideoCodec_H264_SVC and cudaVideoCodec_H264_MVC */
@@ -424,8 +456,6 @@ gst_nv_h264_dec_new_sequence (GstH264Decoder * decoder, const GstH264SPS * sps,
       return FALSE;
     }
 
-    self->last_sps = NULL;
-    self->last_pps = NULL;
     memset (&self->params, 0, sizeof (CUVIDPICPARAMS));
   }
 
@@ -454,6 +484,27 @@ gst_nv_h264_dec_new_picture (GstH264Decoder * decoder,
   return TRUE;
 }
 
+static gboolean
+gst_nv_h264_dec_new_field_picture (GstH264Decoder * decoder,
+    const GstH264Picture * first_field, GstH264Picture * second_field)
+{
+  GstNvDecoderFrame *nv_frame;
+
+  nv_frame = (GstNvDecoderFrame *)
+      gst_h264_picture_get_user_data ((GstH264Picture *) first_field);
+  if (!nv_frame) {
+    GST_ERROR_OBJECT (decoder,
+        "No decoder frame in the first picture %p", first_field);
+    return FALSE;
+  }
+
+  gst_h264_picture_set_user_data (second_field,
+      gst_nv_decoder_frame_ref (nv_frame),
+      (GDestroyNotify) gst_nv_decoder_frame_unref);
+
+  return TRUE;
+}
+
 static GstFlowReturn
 gst_nv_h264_dec_output_picture (GstH264Decoder * decoder,
     GstVideoCodecFrame * frame, GstH264Picture * picture)
@@ -478,13 +529,24 @@ gst_nv_h264_dec_output_picture (GstH264Decoder * decoder,
     goto error;
   }
 
+  if (picture->buffer_flags != 0) {
+    gboolean interlaced =
+        (picture->buffer_flags & GST_VIDEO_BUFFER_FLAG_INTERLACED) != 0;
+    gboolean tff = (picture->buffer_flags & GST_VIDEO_BUFFER_FLAG_TFF) != 0;
+
+    GST_TRACE_OBJECT (self,
+        "apply buffer flags 0x%x (interlaced %d, top-field-first %d)",
+        picture->buffer_flags, interlaced, tff);
+    GST_BUFFER_FLAG_SET (frame->output_buffer, picture->buffer_flags);
+  }
+
   gst_h264_picture_unref (picture);
 
   return gst_video_decoder_finish_frame (vdec, frame);
 
 error:
-  gst_video_decoder_drop_frame (vdec, frame);
   gst_h264_picture_unref (picture);
+  gst_video_decoder_release_frame (vdec, frame);
 
   return GST_FLOW_ERROR;
 }
@@ -531,7 +593,7 @@ gst_nv_h264_dec_picture_params_from_sps (GstNvH264Dec * self,
     const GstH264SPS * sps, gboolean field_pic, CUVIDH264PICPARAMS * params)
 {
   params->residual_colour_transform_flag = sps->separate_colour_plane_flag;
-  params->MbaffFrameFlag = sps->mb_adaptive_frame_field_flag && field_pic;
+  params->MbaffFrameFlag = sps->mb_adaptive_frame_field_flag && !field_pic;
 
 #define COPY_FIELD(f) \
   (params)->f = (sps)->f
@@ -591,6 +653,67 @@ gst_nv_h264_dec_reset_bitstream_params (GstNvH264Dec * self)
   self->params.pSliceDataOffsets = NULL;
 }
 
+static void
+gst_nv_h264_dec_fill_dpb (GstNvH264Dec * self, GstH264Picture * ref,
+    CUVIDH264DPBENTRY * dpb)
+{
+  GstNvDecoderFrame *frame;
+
+  dpb->not_existing = ref->nonexisting;
+  dpb->PicIdx = -1;
+
+  frame = gst_nv_h264_dec_get_decoder_frame_from_picture (self, ref);
+  if (!frame) {
+    dpb->not_existing = 1;
+  } else if (!dpb->not_existing) {
+    dpb->PicIdx = frame->index;
+  }
+
+  if (dpb->not_existing)
+    return;
+
+  if (GST_H264_PICTURE_IS_LONG_TERM_REF (ref)) {
+    dpb->FrameIdx = ref->long_term_frame_idx;
+    dpb->is_long_term = 1;
+  } else {
+    dpb->FrameIdx = ref->frame_num;
+    dpb->is_long_term = 0;
+  }
+
+  switch (ref->field) {
+    case GST_H264_PICTURE_FIELD_FRAME:
+      dpb->FieldOrderCnt[0] = ref->top_field_order_cnt;
+      dpb->FieldOrderCnt[1] = ref->bottom_field_order_cnt;
+      dpb->used_for_reference = 0x3;
+      break;
+    case GST_H264_PICTURE_FIELD_TOP_FIELD:
+      dpb->FieldOrderCnt[0] = ref->top_field_order_cnt;
+      dpb->used_for_reference = 0x1;
+      if (ref->other_field) {
+        dpb->FieldOrderCnt[1] = ref->other_field->bottom_field_order_cnt;
+        dpb->used_for_reference |= 0x2;
+      } else {
+        dpb->FieldOrderCnt[1] = 0;
+      }
+      break;
+    case GST_H264_PICTURE_FIELD_BOTTOM_FIELD:
+      dpb->FieldOrderCnt[1] = ref->bottom_field_order_cnt;
+      dpb->used_for_reference = 0x2;
+      if (ref->other_field) {
+        dpb->FieldOrderCnt[0] = ref->other_field->bottom_field_order_cnt;
+        dpb->used_for_reference |= 0x1;
+      } else {
+        dpb->FieldOrderCnt[0] = 0;
+      }
+      break;
+    default:
+      dpb->FieldOrderCnt[0] = 0;
+      dpb->FieldOrderCnt[1] = 0;
+      dpb->used_for_reference = 0;
+      break;
+  }
+}
+
 static gboolean
 gst_nv_h264_dec_start_picture (GstH264Decoder * decoder,
     GstH264Picture * picture, GstH264Slice * slice, GstH264Dpb * dpb)
@@ -602,8 +725,8 @@ gst_nv_h264_dec_start_picture (GstH264Decoder * decoder,
   const GstH264SPS *sps;
   const GstH264PPS *pps;
   GstNvDecoderFrame *frame;
-  GArray *dpb_array;
+  GArray *ref_list = self->ref_list;
-  gint i;
+  guint i, ref_frame_idx;
 
   g_return_val_if_fail (slice_header->pps != NULL, FALSE);
   g_return_val_if_fail (slice_header->pps->sequence != NULL, FALSE);
@@ -623,14 +746,27 @@ gst_nv_h264_dec_start_picture (GstH264Decoder * decoder,
 
   /* FIXME: update sps/pps related params only when it's required */
   params->PicWidthInMbs = sps->pic_width_in_mbs_minus1 + 1;
+  if (!sps->frame_mbs_only_flag) {
+    params->FrameHeightInMbs = (sps->pic_height_in_map_units_minus1 + 1) << 1;
+  } else {
     params->FrameHeightInMbs = sps->pic_height_in_map_units_minus1 + 1;
+  }
   params->CurrPicIdx = frame->index;
-  /* TODO: verifiy interlaced */
+  params->field_pic_flag = slice_header->field_pic_flag;
-  params->field_pic_flag = picture->field != GST_H264_PICTURE_FIELD_FRAME;
   params->bottom_field_flag =
       picture->field == GST_H264_PICTURE_FIELD_BOTTOM_FIELD;
-  /* TODO: set second_field here */
+  params->second_field = picture->second_field;
-  params->second_field = 0;
+
+  if (picture->field == GST_H264_PICTURE_FIELD_TOP_FIELD) {
+    h264_params->CurrFieldOrderCnt[0] = picture->top_field_order_cnt;
+    h264_params->CurrFieldOrderCnt[1] = 0;
+  } else if (picture->field == GST_H264_PICTURE_FIELD_BOTTOM_FIELD) {
+    h264_params->CurrFieldOrderCnt[0] = 0;
+    h264_params->CurrFieldOrderCnt[1] = picture->bottom_field_order_cnt;
+  } else {
+    h264_params->CurrFieldOrderCnt[0] = picture->top_field_order_cnt;
+    h264_params->CurrFieldOrderCnt[1] = picture->bottom_field_order_cnt;
+  }
 
   /* nBitstreamDataLen, pBitstreamData, nNumSlices and pSliceDataOffsets
    * will be set later */
@@ -642,69 +778,34 @@ gst_nv_h264_dec_start_picture (GstH264Decoder * decoder,
 
   h264_params->frame_num = picture->frame_num;
   h264_params->ref_pic_flag = GST_H264_PICTURE_IS_REF (picture);
-  /* FIXME: should be updated depending on field type? */
-  h264_params->CurrFieldOrderCnt[0] = picture->top_field_order_cnt;
-  h264_params->CurrFieldOrderCnt[1] = picture->bottom_field_order_cnt;
 
-  if (!self->last_sps || self->last_sps != sps) {
-    GST_DEBUG_OBJECT (self, "Update params from SPS and PPS");
   gst_nv_h264_dec_picture_params_from_sps (self,
       sps, slice_header->field_pic_flag, h264_params);
   gst_nv_h264_dec_picture_params_from_pps (self, pps, h264_params);
-    self->last_sps = sps;
+
-    self->last_pps = pps;
+  ref_frame_idx = 0;
-  } else if (!self->last_pps || self->last_pps != pps) {
+  g_array_set_size (ref_list, 0);
-    GST_DEBUG_OBJECT (self, "Update params from PPS");
-    gst_nv_h264_dec_picture_params_from_pps (self, pps, h264_params);
-    self->last_pps = pps;
-  } else {
-    GST_TRACE_OBJECT (self, "SPS and PPS were not updated");
-  }
 
   memset (&h264_params->dpb, 0, sizeof (h264_params->dpb));
-  for (i = 0; i < G_N_ELEMENTS (h264_params->dpb); i++)
+  gst_h264_dpb_get_pictures_short_term_ref (dpb, FALSE, FALSE, ref_list);
+  for (i = 0; ref_frame_idx < 16 && i < ref_list->len; i++) {
+    GstH264Picture *other = g_array_index (ref_list, GstH264Picture *, i);
+    gst_nv_h264_dec_fill_dpb (self, other, &h264_params->dpb[ref_frame_idx]);
+    ref_frame_idx++;
+  }
+  g_array_set_size (ref_list, 0);
+
+  gst_h264_dpb_get_pictures_long_term_ref (dpb, FALSE, ref_list);
+  for (i = 0; ref_frame_idx < 16 && i < ref_list->len; i++) {
+    GstH264Picture *other = g_array_index (ref_list, GstH264Picture *, i);
+    gst_nv_h264_dec_fill_dpb (self, other, &h264_params->dpb[ref_frame_idx]);
+    ref_frame_idx++;
+  }
+  g_array_set_size (ref_list, 0);
+
+  for (i = ref_frame_idx; i < 16; i++)
     h264_params->dpb[i].PicIdx = -1;
 
-  dpb_array = gst_h264_dpb_get_pictures_all (dpb);
-  for (i = 0; i < dpb_array->len && i < G_N_ELEMENTS (h264_params->dpb); i++) {
-    GstH264Picture *other = g_array_index (dpb_array, GstH264Picture *, i);
-    GstNvDecoderFrame *other_frame;
-    gint picture_index = -1;
-    CUVIDH264DPBENTRY *dpb = &h264_params->dpb[i];
-
-    if (!other->ref)
-      continue;
-
-    other_frame = gst_nv_h264_dec_get_decoder_frame_from_picture (self, other);
-
-    if (other_frame)
-      picture_index = other_frame->index;
-
-    dpb->PicIdx = picture_index;
-    if (GST_H264_PICTURE_IS_LONG_TERM_REF (other)) {
-      dpb->FrameIdx = other->long_term_frame_idx;
-      dpb->is_long_term = 1;
-    } else {
-      dpb->FrameIdx = other->frame_num;
-      dpb->is_long_term = 0;
-    }
-
-    dpb->not_existing = other->nonexisting;
-    if (dpb->not_existing && dpb->PicIdx != -1) {
-      GST_WARNING_OBJECT (self,
-          "Non-existing frame has valid picture index %d", dpb->PicIdx);
-      dpb->PicIdx = -1;
-    }
-
-    /* FIXME: 1=top_field, 2=bottom_field, 3=both_fields */
-    dpb->used_for_reference = 3;
-
-    dpb->FieldOrderCnt[0] = other->top_field_order_cnt;
-    dpb->FieldOrderCnt[1] = other->bottom_field_order_cnt;
-  }
-
-  g_array_unref (dpb_array);
-
   return TRUE;
 }