diff --git a/gst-libs/gst/codecs/gsth265decoder.c b/gst-libs/gst/codecs/gsth265decoder.c index 8fd25bedcc..0d63290edb 100644 --- a/gst-libs/gst/codecs/gsth265decoder.c +++ b/gst-libs/gst/codecs/gsth265decoder.c @@ -107,9 +107,6 @@ struct _GstH265DecoderPrivate GArray *ref_pic_list_tmp; GArray *ref_pic_list0; GArray *ref_pic_list1; - - /* Cached array to handle pictures to be outputted */ - GArray *to_output; }; #define parent_class gst_h265_decoder_parent_class @@ -170,11 +167,6 @@ gst_h265_decoder_init (GstH265Decoder * self) sizeof (GstH265Picture *), 32); priv->ref_pic_list1 = g_array_sized_new (FALSE, TRUE, sizeof (GstH265Picture *), 32); - - priv->to_output = g_array_sized_new (FALSE, TRUE, - sizeof (GstH265Picture *), 16); - g_array_set_clear_func (priv->to_output, - (GDestroyNotify) gst_h265_picture_clear); } static void @@ -183,7 +175,6 @@ gst_h265_decoder_finalize (GObject * object) GstH265Decoder *self = GST_H265_DECODER (object); GstH265DecoderPrivate *priv = self->priv; - g_array_unref (priv->to_output); g_array_unref (priv->ref_pic_list_tmp); g_array_unref (priv->ref_pic_list0); g_array_unref (priv->ref_pic_list1); @@ -527,7 +518,7 @@ gst_h265_decoder_preprocess_slice (GstH265Decoder * self, GstH265Slice * slice) if (GST_H265_IS_NAL_TYPE_IDR (nalu->type)) { GST_DEBUG_OBJECT (self, "IDR nalu, clear dpb"); - gst_h265_decoder_drain (GST_VIDEO_DECODER (self)); + gst_h265_decoder_drain_internal (self); } return TRUE; @@ -1289,27 +1280,16 @@ gst_h265_decoder_prepare_rps (GstH265Decoder * self, const GstH265Slice * slice, return TRUE; } -static void -gst_h265_decoder_clear_dpb (GstH265Decoder * self) -{ - GstH265DecoderPrivate *priv = self->priv; - - gst_h265_dpb_clear (priv->dpb); - priv->last_output_poc = -1; -} - static void gst_h265_decoder_do_output_picture (GstH265Decoder * self, - GstH265Picture * picture, gboolean clear_dpb) + GstH265Picture * picture) { GstH265DecoderPrivate *priv = self->priv; GstH265DecoderClass *klass; GstVideoCodecFrame *frame = NULL; - picture->outputted = TRUE; - - if (clear_dpb && !picture->ref) - gst_h265_dpb_delete_by_poc (priv->dpb, picture->pic_order_cnt); + GST_LOG_OBJECT (self, "Output picture %p (poc %d)", picture, + picture->pic_order_cnt); if (picture->pic_order_cnt < priv->last_output_poc) { GST_WARNING_OBJECT (self, @@ -1327,8 +1307,8 @@ gst_h265_decoder_do_output_picture (GstH265Decoder * self, "No available codec frame with frame number %d", picture->system_frame_number); priv->last_ret = GST_FLOW_ERROR; - gst_h265_picture_unref (picture); + gst_h265_picture_unref (picture); return; } @@ -1338,57 +1318,30 @@ gst_h265_decoder_do_output_picture (GstH265Decoder * self, priv->last_ret = klass->output_picture (self, frame, picture); } -static gint -poc_asc_compare (const GstH265Picture ** a, const GstH265Picture ** b) +static void +gst_h265_decoder_clear_dpb (GstH265Decoder * self) { - return (*a)->pic_order_cnt > (*b)->pic_order_cnt; + GstH265DecoderPrivate *priv = self->priv; + + gst_h265_dpb_clear (priv->dpb); + priv->last_output_poc = 0; } static gboolean gst_h265_decoder_drain_internal (GstH265Decoder * self) { GstH265DecoderPrivate *priv = self->priv; - GArray *to_output = priv->to_output; + GstH265Picture *picture; - gst_h265_dpb_delete_outputted (priv->dpb); - gst_h265_dpb_get_pictures_not_outputted (priv->dpb, to_output); - g_array_sort (to_output, (GCompareFunc) poc_asc_compare); + while ((picture = gst_h265_dpb_bump (priv->dpb)) != NULL) + gst_h265_decoder_do_output_picture (self, picture); - while (to_output->len) { - GstH265Picture *picture = g_array_index (to_output, GstH265Picture *, 0); - - /* We want the last reference when outputing so take a ref and then remove - * from both arrays. */ - gst_h265_picture_ref (picture); - g_array_remove_index (to_output, 0); - gst_h265_dpb_delete_by_poc (priv->dpb, picture->pic_order_cnt); - - GST_LOG_OBJECT (self, "Output picture %p (poc %d)", picture, - picture->pic_order_cnt); - gst_h265_decoder_do_output_picture (self, picture, FALSE); - picture = NULL; - } - - g_array_set_size (to_output, 0); gst_h265_dpb_clear (priv->dpb); priv->last_output_poc = 0; + return TRUE; } -static gboolean -gst_h265_decoder_check_latency_count (GArray * array, guint32 max_latency) -{ - gint i; - - for (i = 0; i < array->len; i++) { - GstH265Picture *pic = g_array_index (array, GstH265Picture *, i); - if (!pic->outputted && pic->pic_latency_cnt >= max_latency) - return TRUE; - } - - return FALSE; -} - /* C.5.2.2 */ static gboolean gst_h265_decoder_dpb_init (GstH265Decoder * self, const GstH265Slice * slice, @@ -1397,7 +1350,10 @@ gst_h265_decoder_dpb_init (GstH265Decoder * self, const GstH265Slice * slice, GstH265DecoderPrivate *priv = self->priv; const GstH265SliceHdr *slice_hdr = &slice->header; const GstH265NalUnit *nalu = &slice->nalu; + const GstH265SPS *sps = priv->active_sps; + GstH265Picture *to_output; + /* C 3.2 */ if (GST_H265_IS_NAL_TYPE_IRAP (nalu->type) && picture->NoRaslOutputFlag && !priv->new_bitstream) { if (nalu->type == GST_H265_NAL_SLICE_CRA_NUT) @@ -1408,11 +1364,29 @@ gst_h265_decoder_dpb_init (GstH265Decoder * self, const GstH265Slice * slice, if (picture->NoOutputOfPriorPicsFlag) { GST_DEBUG_OBJECT (self, "Clear dpb"); - gst_h265_decoder_drain (GST_VIDEO_DECODER (self)); + gst_h265_decoder_clear_dpb (self); + } else { + gst_h265_dpb_delete_unused (priv->dpb); + while ((to_output = gst_h265_dpb_bump (priv->dpb)) != NULL) + gst_h265_decoder_do_output_picture (self, to_output); } } else { - /* C 3.2 */ gst_h265_dpb_delete_unused (priv->dpb); + while (gst_h265_dpb_needs_bump (priv->dpb, + sps->max_num_reorder_pics[sps->max_sub_layers_minus1], + priv->SpsMaxLatencyPictures, + sps->max_dec_pic_buffering_minus1[sps->max_sub_layers_minus1] + + 1)) { + to_output = gst_h265_dpb_bump (priv->dpb); + + /* Something wrong... */ + if (!to_output) { + GST_WARNING_OBJECT (self, "Bumping is needed but no picture to output"); + break; + } + + gst_h265_decoder_do_output_picture (self, to_output); + } } return TRUE; @@ -1463,126 +1437,51 @@ static gboolean gst_h265_decoder_finish_picture (GstH265Decoder * self, GstH265Picture * picture) { + GstVideoDecoder *decoder = GST_VIDEO_DECODER (self); GstH265DecoderPrivate *priv = self->priv; const GstH265SPS *sps = priv->active_sps; - GArray *not_outputted = priv->to_output; - guint num_remaining; - gint i; GST_LOG_OBJECT (self, "Finishing picture %p (poc %d), entries in DPB %d", picture, picture->pic_order_cnt, gst_h265_dpb_get_size (priv->dpb)); - /* Get all pictures that haven't been outputted yet */ - gst_h265_dpb_get_pictures_not_outputted (priv->dpb, not_outputted); + gst_h265_dpb_delete_unused (priv->dpb); - /* C.5.2.3 */ - if (picture->output_flag) { - for (i = 0; i < not_outputted->len; i++) { - GstH265Picture *other = - g_array_index (not_outputted, GstH265Picture *, i); + /* This picture is decode only, drop corresponding frame */ + if (!picture->output_flag) { + GstVideoCodecFrame *frame = gst_video_decoder_get_frame (decoder, + picture->system_frame_number); - if (!other->outputted) - other->pic_latency_cnt++; + gst_video_decoder_release_frame (decoder, frame); + } + + /* gst_h265_dpb_add() will take care of pic_latency_cnt increment and + * reference picture marking for this picture */ + gst_h265_dpb_add (priv->dpb, picture); + + /* NOTE: As per C.5.2.2, bumping by sps_max_dec_pic_buffering_minus1 is + * applied only for the output and removal of pictures from the DPB before + * the decoding of the current picture. So pass zero here */ + while (gst_h265_dpb_needs_bump (priv->dpb, + sps->max_num_reorder_pics[sps->max_sub_layers_minus1], + priv->SpsMaxLatencyPictures, 0)) { + GstH265Picture *to_output = gst_h265_dpb_bump (priv->dpb); + + /* Something wrong... */ + if (!to_output) { + GST_WARNING_OBJECT (self, "Bumping is needed but no picture to output"); + break; } - picture->outputted = FALSE; - picture->pic_latency_cnt = 0; - } else { - picture->outputted = TRUE; + gst_h265_decoder_do_output_picture (self, to_output); } - /* set pic as short_term_ref */ - picture->ref = TRUE; - picture->long_term = FALSE; - - /* Include the one we've just decoded */ - if (picture->output_flag) - g_array_append_val (not_outputted, picture); - - /* for debugging */ -#ifndef GST_DISABLE_GST_DEBUG - GST_TRACE_OBJECT (self, "Before sorting not outputted list"); - for (i = 0; i < not_outputted->len; i++) { - GstH265Picture *tmp = g_array_index (not_outputted, GstH265Picture *, i); - GST_TRACE_OBJECT (self, - "\t%dth picture %p (poc %d)", i, tmp, tmp->pic_order_cnt); + if (gst_h265_dpb_is_full (priv->dpb)) { + /* If we haven't managed to output anything to free up space in DPB + * to store this picture, it's an error in the stream */ + GST_WARNING_OBJECT (self, "Could not free up space in DPB"); + return FALSE; } -#endif - - /* Sort in output order */ - g_array_sort (not_outputted, (GCompareFunc) poc_asc_compare); - -#ifndef GST_DISABLE_GST_DEBUG - GST_TRACE_OBJECT (self, - "After sorting not outputted list in poc ascending order"); - for (i = 0; i < not_outputted->len; i++) { - GstH265Picture *tmp = g_array_index (not_outputted, GstH265Picture *, i); - GST_TRACE_OBJECT (self, - "\t%dth picture %p (poc %d)", i, tmp, tmp->pic_order_cnt); - } -#endif - - /* Try to output as many pictures as we can. A picture can be output, - * if the number of decoded and not yet outputted pictures that would remain - * in DPB afterwards would at least be equal to max_num_reorder_frames. - * If the outputted picture is not a reference picture, it doesn't have - * to remain in the DPB and can be removed */ - num_remaining = not_outputted->len; - - while (num_remaining > sps->max_num_reorder_pics[sps->max_sub_layers_minus1] - || (num_remaining && - sps->max_latency_increase_plus1[sps->max_sub_layers_minus1] && - gst_h265_decoder_check_latency_count (not_outputted, - priv->SpsMaxLatencyPictures))) { - gboolean clear_dpb = TRUE; - GstH265Picture *to_output = - g_array_index (not_outputted, GstH265Picture *, 0); - - gst_h265_picture_ref (to_output); - g_array_remove_index (not_outputted, 0); - - GST_LOG_OBJECT (self, - "Output picture %p (poc %d)", to_output, to_output->pic_order_cnt); - - /* Current picture hasn't been inserted into DPB yet, so don't remove it - * if we managed to output it immediately */ - if (picture && to_output == picture) { - clear_dpb = FALSE; - - if (picture->ref) { - GST_TRACE_OBJECT (self, - "Put current picture %p (poc %d) to dpb", - picture, picture->pic_order_cnt); - gst_h265_dpb_add (priv->dpb, gst_h265_picture_ref (picture)); - } - - /* and mark current picture as handled */ - picture = NULL; - } - - gst_h265_decoder_do_output_picture (self, to_output, clear_dpb); - - num_remaining--; - } - - /* If we haven't managed to output the picture that we just decoded, or if - * it's a reference picture, we have to store it in DPB */ - if (picture && (!picture->outputted || picture->ref)) { - if (gst_h265_dpb_is_full (priv->dpb)) { - /* If we haven't managed to output anything to free up space in DPB - * to store this picture, it's an error in the stream */ - GST_WARNING_OBJECT (self, "Could not free up space in DPB"); - return FALSE; - } - - GST_TRACE_OBJECT (self, - "Put picture %p (outputted %d, ref %d, poc %d) to dpb", - picture, picture->outputted, picture->ref, picture->pic_order_cnt); - gst_h265_dpb_add (priv->dpb, gst_h265_picture_ref (picture)); - } - - g_array_set_size (not_outputted, 0); return TRUE; } diff --git a/gst-libs/gst/codecs/gsth265picture.c b/gst-libs/gst/codecs/gsth265picture.c index 69cd3556c8..573bcf70cb 100644 --- a/gst-libs/gst/codecs/gsth265picture.c +++ b/gst-libs/gst/codecs/gsth265picture.c @@ -105,6 +105,7 @@ struct _GstH265Dpb { GArray *pic_list; gint max_num_pics; + gint num_output_needed; }; /** @@ -187,6 +188,7 @@ gst_h265_dpb_clear (GstH265Dpb * dpb) g_return_if_fail (dpb != NULL); g_array_set_size (dpb->pic_list, 0); + dpb->num_output_needed = 0; } /** @@ -194,7 +196,8 @@ gst_h265_dpb_clear (GstH265Dpb * dpb) * @dpb: a #GstH265Dpb * @picture: (transfer full): a #GstH265Picture * - * Store the @picture + * Store the @picture and perform increase pic_latency_cnt as defined in + * "C.5.2.3 Additional bumping" process */ void gst_h265_dpb_add (GstH265Dpb * dpb, GstH265Picture * picture) @@ -202,6 +205,27 @@ gst_h265_dpb_add (GstH265Dpb * dpb, GstH265Picture * picture) g_return_if_fail (dpb != NULL); g_return_if_fail (GST_IS_H265_PICTURE (picture)); + if (picture->output_flag) { + gint i; + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH265Picture *other = + g_array_index (dpb->pic_list, GstH265Picture *, i); + + if (other->needed_for_output) + other->pic_latency_cnt++; + } + + dpb->num_output_needed++; + picture->needed_for_output = TRUE; + } else { + picture->needed_for_output = FALSE; + } + + /* C.3.4 */ + picture->ref = TRUE; + picture->long_term = FALSE; + g_array_append_val (dpb->pic_list, picture); } @@ -209,7 +233,7 @@ gst_h265_dpb_add (GstH265Dpb * dpb, GstH265Picture * picture) * gst_h265_dpb_delete_unused: * @dpb: a #GstH265Dpb * - * Delete already outputted and not referenced all pictures from dpb + * Delete not needed for output and not referenced all pictures from dpb */ void gst_h265_dpb_delete_unused (GstH265Dpb * dpb) @@ -222,7 +246,7 @@ gst_h265_dpb_delete_unused (GstH265Dpb * dpb) GstH265Picture *picture = g_array_index (dpb->pic_list, GstH265Picture *, i); - if (picture->outputted && !picture->ref) { + if (!picture->needed_for_output && !picture->ref) { GST_TRACE ("remove picture %p (poc %d) from dpb", picture, picture->pic_order_cnt); g_array_remove_index (dpb->pic_list, i); @@ -231,61 +255,6 @@ gst_h265_dpb_delete_unused (GstH265Dpb * dpb) } } -/** - * gst_h265_dpb_delete_outputted: - * @dpb: a #GstH265Dpb - * - * Delete already outputted picture, even if they are referenced. - * - * Since: 1.20 - */ -void -gst_h265_dpb_delete_outputted (GstH265Dpb * dpb) -{ - gint i; - - g_return_if_fail (dpb != NULL); - - for (i = 0; i < dpb->pic_list->len; i++) { - GstH265Picture *picture = - g_array_index (dpb->pic_list, GstH265Picture *, i); - - if (picture->outputted) { - GST_TRACE ("remove picture %p (poc %d) from dpb", - picture, picture->pic_order_cnt); - g_array_remove_index_fast (dpb->pic_list, i); - i--; - } - } -} - -/** - * gst_h265_dpb_delete_by_poc: - * @dpb: a #GstH265Dpb - * @poc: a poc of #GstH265Picture to remove - * - * Delete a #GstH265Dpb by @poc - */ -void -gst_h265_dpb_delete_by_poc (GstH265Dpb * dpb, gint poc) -{ - gint i; - - g_return_if_fail (dpb != NULL); - - for (i = 0; i < dpb->pic_list->len; i++) { - GstH265Picture *picture = - g_array_index (dpb->pic_list, GstH265Picture *, i); - - if (picture->pic_order_cnt == poc) { - g_array_remove_index (dpb->pic_list, i); - return; - } - } - - GST_WARNING ("Couldn't find picture with poc %d", poc); -} - /** * gst_h265_dpb_num_ref_pictures: * @dpb: a #GstH265Dpb @@ -448,33 +417,6 @@ gst_h265_dpb_get_long_ref_by_poc (GstH265Dpb * dpb, gint poc) return NULL; } -/** - * gst_h265_dpb_get_pictures_not_outputted: - * @dpb: a #GstH265Dpb - * @out: (out) (element-type GstH265Picture) (transfer full): an array - * of #GstH265Picture pointer - * - * Retrieve all not-outputted pictures from @dpb - */ -void -gst_h265_dpb_get_pictures_not_outputted (GstH265Dpb * dpb, GArray * out) -{ - gint i; - - g_return_if_fail (dpb != NULL); - g_return_if_fail (out != NULL); - - for (i = 0; i < dpb->pic_list->len; i++) { - GstH265Picture *picture = - g_array_index (dpb->pic_list, GstH265Picture *, i); - - if (!picture->outputted) { - gst_h265_picture_ref (picture); - g_array_append_val (out, picture); - } - } -} - /** * gst_h265_dpb_get_pictures_all: * @dpb: a #GstH265Dpb @@ -548,3 +490,136 @@ gst_h265_dpb_get_picture (GstH265Dpb * dpb, guint32 system_frame_number) return NULL; } + +static gboolean +gst_h265_decoder_check_latency_count (GstH265Dpb * dpb, guint32 max_latency) +{ + gint i; + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH265Picture *picture = + g_array_index (dpb->pic_list, GstH265Picture *, i); + + if (!picture->needed_for_output) + continue; + + if (picture->pic_latency_cnt >= max_latency) + return TRUE; + } + + return FALSE; +} + +/** + * gst_h265_dpb_needs_bump: + * @dpb: a #GstH265Dpb + * @max_num_reorder_pics: sps_max_num_reorder_pics[HighestTid] + * @max_latency_increase: SpsMaxLatencyPictures[HighestTid] + * @max_dec_pic_buffering: sps_max_dec_pic_buffering_minus1[HighestTid ] + 1 + * or zero if this shouldn't be used for bumping decision + * + * Returns: %TRUE if bumping is required + * + * Since: 1.20 + */ +gboolean +gst_h265_dpb_needs_bump (GstH265Dpb * dpb, guint max_num_reorder_pics, + guint max_latency_increase, guint max_dec_pic_buffering) +{ + g_return_val_if_fail (dpb != NULL, FALSE); + g_assert (dpb->num_output_needed >= 0); + + /* C.5.2.3 */ + if (dpb->num_output_needed > max_num_reorder_pics) { + GST_TRACE ("num_output_needed (%d) > max_num_reorder_pics (%d)", + dpb->num_output_needed, max_num_reorder_pics); + return TRUE; + } + + if (dpb->num_output_needed && max_latency_increase && + gst_h265_decoder_check_latency_count (dpb, max_latency_increase)) { + GST_TRACE ("has late picture, max_latency_increase: %d", + max_latency_increase); + return TRUE; + } + + /* C.5.2.2 */ + if (max_dec_pic_buffering && dpb->pic_list->len >= max_dec_pic_buffering) { + GST_TRACE ("dpb size (%d) >= max_dec_pic_buffering (%d)", + dpb->pic_list->len, max_dec_pic_buffering); + return TRUE; + } + + return FALSE; +} + +static gint +gst_h265_dpb_get_lowest_output_needed_picture (GstH265Dpb * dpb, + GstH265Picture ** picture) +{ + gint i; + GstH265Picture *lowest = NULL; + gint index = -1; + + *picture = NULL; + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH265Picture *picture = + g_array_index (dpb->pic_list, GstH265Picture *, i); + + if (!picture->needed_for_output) + continue; + + if (!lowest) { + lowest = picture; + index = i; + continue; + } + + if (picture->pic_order_cnt < lowest->pic_order_cnt) { + lowest = picture; + index = i; + } + } + + if (lowest) + *picture = gst_h265_picture_ref (lowest); + + return index; +} + +/** + * gst_h265_dpb_bump: + * @dpb: a #GstH265Dpb + * + * Perform bumping process as defined in C.5.2.4 "Bumping" process. + * + * Returns: (nullable) (transfer full): a #GstH265Picture which is needed to be + * outputted + * + * Since: 1.20 + */ +GstH265Picture * +gst_h265_dpb_bump (GstH265Dpb * dpb) +{ + GstH265Picture *picture; + gint index; + + g_return_val_if_fail (dpb != NULL, NULL); + + /* C.5.2.4 "Bumping" process */ + index = gst_h265_dpb_get_lowest_output_needed_picture (dpb, &picture); + + if (!picture || index < 0) + return NULL; + + picture->needed_for_output = FALSE; + + dpb->num_output_needed--; + g_assert (dpb->num_output_needed >= 0); + + if (!picture->ref) + g_array_remove_index_fast (dpb->pic_list, index); + + return picture; +} diff --git a/gst-libs/gst/codecs/gsth265picture.h b/gst-libs/gst/codecs/gsth265picture.h index b1155ceeb8..281e4467b7 100644 --- a/gst-libs/gst/codecs/gsth265picture.h +++ b/gst-libs/gst/codecs/gsth265picture.h @@ -59,6 +59,7 @@ typedef enum struct _GstH265Picture { + /*< private >*/ GstMiniObject parent; GstH265SliceType type; @@ -81,7 +82,7 @@ struct _GstH265Picture gboolean ref; gboolean long_term; - gboolean outputted; + gboolean needed_for_output; GstH265PictureField field; @@ -160,13 +161,6 @@ void gst_h265_dpb_add (GstH265Dpb * dpb, GST_CODECS_API void gst_h265_dpb_delete_unused (GstH265Dpb * dpb); -GST_CODECS_API -void gst_h265_dpb_delete_outputted (GstH265Dpb * dpb); - -GST_CODECS_API -void gst_h265_dpb_delete_by_poc (GstH265Dpb * dpb, - gint poc); - GST_CODECS_API gint gst_h265_dpb_num_ref_pictures (GstH265Dpb * dpb); @@ -189,10 +183,6 @@ GST_CODECS_API GstH265Picture * gst_h265_dpb_get_long_ref_by_poc (GstH265Dpb * dpb, gint poc); -GST_CODECS_API -void gst_h265_dpb_get_pictures_not_outputted (GstH265Dpb * dpb, - GArray * out); - GST_CODECS_API GArray * gst_h265_dpb_get_pictures_all (GstH265Dpb * dpb); @@ -206,6 +196,15 @@ gint gst_h265_dpb_get_size (GstH265Dpb * dpb); GST_CODECS_API gboolean gst_h265_dpb_is_full (GstH265Dpb * dpb); +GST_CODECS_API +gboolean gst_h265_dpb_needs_bump (GstH265Dpb * dpb, + guint max_num_reorder_pics, + guint max_latency_increase, + guint max_dec_pic_buffering); + +GST_CODECS_API +GstH265Picture * gst_h265_dpb_bump (GstH265Dpb * dpb); + G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstH265Picture, gst_h265_picture_unref) G_END_DECLS