From 64ea751cd2159f93112377d22d06f80d1bb7919f Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Sun, 15 Nov 2020 00:20:54 +0900 Subject: [PATCH] codecs: h264decoder: Add support for field ref picture list modification Part-of: --- gst-libs/gst/codecs/gsth264decoder.c | 272 ++++++++++++++++++++++++++- gst-libs/gst/codecs/gsth264decoder.h | 5 + 2 files changed, 275 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/codecs/gsth264decoder.c b/gst-libs/gst/codecs/gsth264decoder.c index b336c19674..a039c065a5 100644 --- a/gst-libs/gst/codecs/gsth264decoder.c +++ b/gst-libs/gst/codecs/gsth264decoder.c @@ -137,6 +137,13 @@ struct _GstH264DecoderPrivate GArray *ref_pic_list_b0; GArray *ref_pic_list_b1; + /* Temporary picture list, for reference picture lists in fields, + * corresponding to 8.2.4.2.2 refFrameList0ShortTerm, refFrameList0LongTerm + * and 8.2.4.2.5 refFrameList1ShortTerm and refFrameListLongTerm */ + GArray *ref_frame_list_0_short_term; + GArray *ref_frame_list_1_short_term; + GArray *ref_frame_list_long_term; + /* Reference picture lists, constructed for each slice */ GArray *ref_pic_list0; GArray *ref_pic_list1; @@ -231,6 +238,21 @@ gst_h264_decoder_init (GstH264Decoder * self) g_array_set_clear_func (priv->ref_pic_list_b1, (GDestroyNotify) gst_h264_picture_clear); + priv->ref_frame_list_0_short_term = g_array_sized_new (FALSE, TRUE, + sizeof (GstH264Picture *), 32); + g_array_set_clear_func (priv->ref_frame_list_0_short_term, + (GDestroyNotify) gst_h264_picture_clear); + + priv->ref_frame_list_1_short_term = g_array_sized_new (FALSE, TRUE, + sizeof (GstH264Picture *), 32); + g_array_set_clear_func (priv->ref_frame_list_1_short_term, + (GDestroyNotify) gst_h264_picture_clear); + + priv->ref_frame_list_long_term = g_array_sized_new (FALSE, TRUE, + sizeof (GstH264Picture *), 32); + g_array_set_clear_func (priv->ref_frame_list_long_term, + (GDestroyNotify) gst_h264_picture_clear); + priv->ref_pic_list0 = g_array_sized_new (FALSE, TRUE, sizeof (GstH264Picture *), 32); priv->ref_pic_list1 = g_array_sized_new (FALSE, TRUE, @@ -246,6 +268,9 @@ gst_h264_decoder_finalize (GObject * object) g_array_unref (priv->ref_pic_list_p0); g_array_unref (priv->ref_pic_list_b0); g_array_unref (priv->ref_pic_list_b1); + g_array_unref (priv->ref_frame_list_0_short_term); + g_array_unref (priv->ref_frame_list_1_short_term); + g_array_unref (priv->ref_frame_list_long_term); g_array_unref (priv->ref_pic_list0); g_array_unref (priv->ref_pic_list1); @@ -2152,6 +2177,139 @@ construct_ref_pic_lists_p (GstH264Decoder * self, #endif } +static gint +frame_num_wrap_desc_compare (const GstH264Picture ** a, + const GstH264Picture ** b) +{ + return (*b)->frame_num_wrap - (*a)->frame_num_wrap; +} + +static gint +long_term_frame_idx_asc_compare (const GstH264Picture ** a, + const GstH264Picture ** b) +{ + return (*a)->long_term_frame_idx - (*b)->long_term_frame_idx; +} + +/* init_picture_refs_fields_1 in gstvaapidecoder_h264.c */ +static void +init_picture_refs_fields_1 (GstH264Decoder * self, GstH264PictureField field, + GArray * ref_frame_list, GArray * ref_pic_list_x) +{ + guint i = 0, j = 0; + + do { + for (; i < ref_frame_list->len; i++) { + GstH264Picture *pic = g_array_index (ref_frame_list, GstH264Picture *, i); + if (pic->field == field) { + pic = gst_h264_picture_ref (pic); + g_array_append_val (ref_pic_list_x, pic); + i++; + break; + } + } + + for (; j < ref_frame_list->len; j++) { + GstH264Picture *pic = g_array_index (ref_frame_list, GstH264Picture *, j); + if (pic->field != field) { + pic = gst_h264_picture_ref (pic); + g_array_append_val (ref_pic_list_x, pic); + j++; + break; + } + } + } while (i < ref_frame_list->len || j < ref_frame_list->len); +} + +static void +construct_ref_field_pic_lists_p (GstH264Decoder * self, + GstH264Picture * current_picture) +{ + GstH264DecoderPrivate *priv = self->priv; + gint pos; + + g_array_set_size (priv->ref_pic_list_p0, 0); + g_array_set_size (priv->ref_frame_list_0_short_term, 0); + g_array_set_size (priv->ref_frame_list_long_term, 0); + + /* 8.2.4.2.2, 8.2.4.2.5 refFrameList0ShortTerm: + * short-term ref pictures sorted by descending frame_num_wrap. + */ + gst_h264_dpb_get_pictures_short_term_ref (priv->dpb, + TRUE, TRUE, priv->ref_frame_list_0_short_term); + g_array_sort (priv->ref_frame_list_0_short_term, + (GCompareFunc) frame_num_wrap_desc_compare); + +#ifndef GST_DISABLE_GST_DEBUG + if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE + && priv->ref_frame_list_0_short_term->len) { + GString *str = g_string_new (NULL); + for (pos = 0; pos < priv->ref_frame_list_0_short_term->len; pos++) { + GstH264Picture *ref = g_array_index (priv->ref_frame_list_0_short_term, + GstH264Picture *, pos); + g_string_append_printf (str, "|%i(%d)", ref->frame_num_wrap, ref->field); + } + GST_TRACE_OBJECT (self, "ref_frame_list_0_short_term (%d): %s|", + current_picture->field, str->str); + g_string_free (str, TRUE); + } +#endif + + /* 8.2.4.2.2 refFrameList0LongTerm,: + * long-term ref pictures sorted by ascending long_term_frame_idx. + */ + gst_h264_dpb_get_pictures_long_term_ref (priv->dpb, + TRUE, priv->ref_frame_list_long_term); + g_array_sort (priv->ref_frame_list_long_term, + (GCompareFunc) long_term_frame_idx_asc_compare); + +#ifndef GST_DISABLE_GST_DEBUG + if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE + && priv->ref_frame_list_long_term->len) { + GString *str = g_string_new (NULL); + for (pos = 0; pos < priv->ref_frame_list_long_term->len; pos++) { + GstH264Picture *ref = g_array_index (priv->ref_frame_list_0_short_term, + GstH264Picture *, pos); + g_string_append_printf (str, "|%i(%d)", ref->long_term_frame_idx, + ref->field); + } + GST_TRACE_OBJECT (self, "ref_frame_list_0_long_term (%d): %s|", + current_picture->field, str->str); + g_string_free (str, TRUE); + } +#endif + + /* 8.2.4.2.5 */ + init_picture_refs_fields_1 (self, current_picture->field, + priv->ref_frame_list_0_short_term, priv->ref_pic_list_p0); + init_picture_refs_fields_1 (self, current_picture->field, + priv->ref_frame_list_long_term, priv->ref_pic_list_p0); + +#ifndef GST_DISABLE_GST_DEBUG + if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_DEBUG + && priv->ref_pic_list_p0->len) { + GString *str = g_string_new (NULL); + for (pos = 0; pos < priv->ref_pic_list_p0->len; pos++) { + GstH264Picture *ref = + g_array_index (priv->ref_pic_list_p0, GstH264Picture *, pos); + if (!GST_H264_PICTURE_IS_LONG_TERM_REF (ref)) + g_string_append_printf (str, "|%i(%d)s", ref->frame_num_wrap, + ref->field); + else + g_string_append_printf (str, "|%i(%d)l", ref->long_term_frame_idx, + ref->field); + } + GST_DEBUG_OBJECT (self, "ref_pic_list_p0 (%d): %s|", current_picture->field, + str->str); + g_string_free (str, TRUE); + } +#endif + + /* Clear temporary lists, now pictures are owned by ref_pic_list_p0 */ + g_array_set_size (priv->ref_frame_list_0_short_term, 0); + g_array_set_size (priv->ref_frame_list_long_term, 0); +} + static gboolean lists_are_equal (GArray * l1, GArray * l2) { @@ -2300,6 +2458,111 @@ construct_ref_pic_lists_b (GstH264Decoder * self, print_ref_pic_list_b (self, priv->ref_pic_list_b1, 1); } +static void +construct_ref_field_pic_lists_b (GstH264Decoder * self, + GstH264Picture * current_picture) +{ + GstH264DecoderPrivate *priv = self->priv; + gint pos; + + /* refFrameList0ShortTerm (8.2.4.2.4) [[1] [2]], where: + * [1] shortterm ref pics with POC < current_picture's POC sorted by descending POC, + * [2] shortterm ref pics with POC > current_picture's POC by ascending POC, + */ + g_array_set_size (priv->ref_pic_list_b0, 0); + g_array_set_size (priv->ref_pic_list_b1, 0); + g_array_set_size (priv->ref_frame_list_0_short_term, 0); + g_array_set_size (priv->ref_frame_list_1_short_term, 0); + g_array_set_size (priv->ref_frame_list_long_term, 0); + + /* 8.2.4.2.4 + * When pic_order_cnt_type is equal to 0, reference pictures that are marked + * as "non-existing" as specified in clause 8.2.5.2 are not included in either + * RefPicList0 or RefPicList1 + */ + gst_h264_dpb_get_pictures_short_term_ref (priv->dpb, + current_picture->pic_order_cnt_type != 0, TRUE, + priv->ref_frame_list_0_short_term); + + /* First sort ascending, this will put [1] in right place and finish + * [2]. */ + print_ref_pic_list_b (self, priv->ref_frame_list_0_short_term, 0); + g_array_sort (priv->ref_pic_list_b0, (GCompareFunc) poc_asc_compare); + print_ref_pic_list_b (self, priv->ref_frame_list_0_short_term, 0); + + /* Find first with POC > current_picture's POC to get first element + * in [2]... */ + pos = split_ref_pic_list_b (self, priv->ref_frame_list_0_short_term, + (GCompareFunc) poc_asc_compare); + + GST_DEBUG_OBJECT (self, "split point %i", pos); + + /* and sort [1] descending, thus finishing sequence [1] [2]. */ + g_qsort_with_data (priv->ref_frame_list_0_short_term->data, pos, + sizeof (gpointer), (GCompareDataFunc) poc_desc_compare, NULL); + + /* refFrameList1ShortTerm (8.2.4.2.4) [[1] [2]], where: + * [1] shortterm ref pics with POC > curr_pic's POC sorted by ascending POC, + * [2] shortterm ref pics with POC < curr_pic's POC by descending POC, + */ + gst_h264_dpb_get_pictures_short_term_ref (priv->dpb, + current_picture->pic_order_cnt_type != 0, TRUE, + priv->ref_frame_list_1_short_term); + + /* First sort by descending POC. */ + g_array_sort (priv->ref_frame_list_1_short_term, + (GCompareFunc) poc_desc_compare); + + /* Split at first with POC < current_picture's POC to get first element + * in [2]... */ + pos = split_ref_pic_list_b (self, priv->ref_frame_list_1_short_term, + (GCompareFunc) poc_desc_compare); + + /* and sort [1] ascending. */ + g_qsort_with_data (priv->ref_frame_list_1_short_term->data, pos, + sizeof (gpointer), (GCompareDataFunc) poc_asc_compare, NULL); + + /* 8.2.4.2.2 refFrameList0LongTerm,: + * long-term ref pictures sorted by ascending long_term_frame_idx. + */ + gst_h264_dpb_get_pictures_long_term_ref (priv->dpb, + TRUE, priv->ref_frame_list_long_term); + g_array_sort (priv->ref_frame_list_long_term, + (GCompareFunc) long_term_frame_idx_asc_compare); + + /* 8.2.4.2.5 RefPicList0 */ + init_picture_refs_fields_1 (self, current_picture->field, + priv->ref_frame_list_0_short_term, priv->ref_pic_list_b0); + init_picture_refs_fields_1 (self, current_picture->field, + priv->ref_frame_list_long_term, priv->ref_pic_list_b0); + + /* 8.2.4.2.5 RefPicList1 */ + init_picture_refs_fields_1 (self, current_picture->field, + priv->ref_frame_list_1_short_term, priv->ref_pic_list_b1); + init_picture_refs_fields_1 (self, current_picture->field, + priv->ref_frame_list_long_term, priv->ref_pic_list_b1); + + /* If lists identical, swap first two entries in RefPicList1 (spec + * 8.2.4.2.5) */ + if (priv->ref_pic_list_b1->len > 1 + && lists_are_equal (priv->ref_pic_list_b0, priv->ref_pic_list_b1)) { + /* swap */ + GstH264Picture **list = (GstH264Picture **) priv->ref_pic_list_b1->data; + GstH264Picture *pic = list[0]; + list[0] = list[1]; + list[1] = pic; + } + + print_ref_pic_list_b (self, priv->ref_pic_list_b0, 0); + print_ref_pic_list_b (self, priv->ref_pic_list_b1, 1); + + /* Clear temporary lists, now pictures are owned by ref_pic_list_b0 + * and ref_pic_list_b1 */ + g_array_set_size (priv->ref_frame_list_0_short_term, 0); + g_array_set_size (priv->ref_frame_list_1_short_term, 0); + g_array_set_size (priv->ref_frame_list_long_term, 0); +} + static void gst_h264_decoder_prepare_ref_pic_lists (GstH264Decoder * self, GstH264Picture * current_picture) @@ -2329,8 +2592,13 @@ gst_h264_decoder_prepare_ref_pic_lists (GstH264Decoder * self, return; } - construct_ref_pic_lists_p (self, current_picture); - construct_ref_pic_lists_b (self, current_picture); + if (GST_H264_PICTURE_IS_FRAME (current_picture)) { + construct_ref_pic_lists_p (self, current_picture); + construct_ref_pic_lists_b (self, current_picture); + } else { + construct_ref_field_pic_lists_p (self, current_picture); + construct_ref_field_pic_lists_b (self, current_picture); + } } static void diff --git a/gst-libs/gst/codecs/gsth264decoder.h b/gst-libs/gst/codecs/gsth264decoder.h index d2e3948c26..dfe5d2f3f4 100644 --- a/gst-libs/gst/codecs/gsth264decoder.h +++ b/gst-libs/gst/codecs/gsth264decoder.h @@ -138,6 +138,11 @@ struct _GstH264DecoderClass * for subclass to decode it. If gst_h264_decoder_set_process_ref_pic_lists() * is called with %TRUE by the subclass, @ref_pic_list0 and @ref_pic_list1 * are non-%NULL. + * In case of interlaced stream, @ref_pic_list0 and @ref_pic_list1 will + * contain only the first field of complementary reference field pair + * if currently being decoded picture is a frame picture. Subclasses might + * need to retrive the other field (i.e., the second field) of the picture + * if needed. */ gboolean (*decode_slice) (GstH264Decoder * decoder, GstH264Picture * picture,