mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2025-01-25 16:48:11 +00:00
Code Issues 1 Projects Releases Wiki Activity

codecs: h264decoder: Rework for DPB management

Browse source 
Sync with recent h265decoder DPB implementation.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1761>
This commit is contained in:
Seungha Yang 2020-11-02 00:08:04 +09:00
parent b70ceb4235
commit 5527cc4a2e
3 changed files with 454 additions and 436 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

414
gst-libs/gst/codecs/gsth264decoder.c
View file

@ -140,9 +140,6 @@ struct _GstH264DecoderPrivate
/* Reference picture lists, constructed for each slice */ /* Reference picture lists, constructed for each slice */
GArray *ref_pic_list0; GArray *ref_pic_list0;
GArray *ref_pic_list1; GArray *ref_pic_list1;
/* Cached array to handle pictures to be outputted */
GArray *to_output;
}; };
#define parent_class gst_h264_decoder_parent_class #define parent_class gst_h264_decoder_parent_class
@ -183,6 +180,10 @@ static gboolean gst_h264_decoder_finish_picture (GstH264Decoder * self,
static void gst_h264_decoder_prepare_ref_pic_lists (GstH264Decoder * self); static void gst_h264_decoder_prepare_ref_pic_lists (GstH264Decoder * self);
static void gst_h264_decoder_clear_ref_pic_lists (GstH264Decoder * self); static void gst_h264_decoder_clear_ref_pic_lists (GstH264Decoder * self);
static gboolean gst_h264_decoder_modify_ref_pic_lists (GstH264Decoder * self); static gboolean gst_h264_decoder_modify_ref_pic_lists (GstH264Decoder * self);
static gboolean
gst_h264_decoder_sliding_window_picture_marking (GstH264Decoder * self);
static void gst_h264_decoder_do_output_picture (GstH264Decoder * self,
GstH264Picture * picture);
static void static void
gst_h264_decoder_class_init (GstH264DecoderClass * klass) gst_h264_decoder_class_init (GstH264DecoderClass * klass)
@ -230,11 +231,6 @@ gst_h264_decoder_init (GstH264Decoder * self)
sizeof (GstH264Picture *), 32); sizeof (GstH264Picture *), 32);
priv->ref_pic_list1 = g_array_sized_new (FALSE, TRUE, priv->ref_pic_list1 = g_array_sized_new (FALSE, TRUE,
sizeof (GstH264Picture *), 32); sizeof (GstH264Picture *), 32);
priv->to_output = g_array_sized_new (FALSE, TRUE,
sizeof (GstH264Picture *), 16);
g_array_set_clear_func (priv->to_output,
(GDestroyNotify) gst_h264_picture_clear);
} }
static void static void
@ -248,7 +244,6 @@ gst_h264_decoder_finalize (GObject * object)
g_array_unref (priv->ref_pic_list_b1); g_array_unref (priv->ref_pic_list_b1);
g_array_unref (priv->ref_pic_list0); g_array_unref (priv->ref_pic_list0);
g_array_unref (priv->ref_pic_list1); g_array_unref (priv->ref_pic_list1);
g_array_unref (priv->to_output);
G_OBJECT_CLASS (parent_class)->finalize (object); G_OBJECT_CLASS (parent_class)->finalize (object);
} }
@ -292,13 +287,28 @@ gst_h264_decoder_stop (GstVideoDecoder * decoder)
} }
static void static void
gst_h264_decoder_clear_dpb (GstH264Decoder * self) gst_h264_decoder_clear_dpb (GstH264Decoder * self, gboolean flush)
{ {
GstVideoDecoder *decoder = GST_VIDEO_DECODER (self);
GstH264DecoderPrivate *priv = self->priv; GstH264DecoderPrivate *priv = self->priv;
GstH264Picture *picture;
/* If we are not flushing now, videodecoder baseclass will hold
* GstVideoCodecFrame. Release frames manually */
if (!flush) {
while ((picture = gst_h264_dpb_bump (priv->dpb, TRUE)) != NULL) {
GstVideoCodecFrame *frame = gst_video_decoder_get_frame (decoder,
picture->system_frame_number);
if (frame)
gst_video_decoder_release_frame (decoder, frame);
gst_h264_picture_unref (picture);
}
}
gst_h264_decoder_clear_ref_pic_lists (self); gst_h264_decoder_clear_ref_pic_lists (self);
gst_h264_dpb_clear (priv->dpb); gst_h264_dpb_clear (priv->dpb);
priv->last_output_poc = -1; priv->last_output_poc = 0;
} }
static gboolean static gboolean
@ -306,7 +316,7 @@ gst_h264_decoder_flush (GstVideoDecoder * decoder)
{ {
GstH264Decoder *self = GST_H264_DECODER (decoder); GstH264Decoder *self = GST_H264_DECODER (decoder);
gst_h264_decoder_clear_dpb (self); gst_h264_decoder_clear_dpb (self, TRUE);
return TRUE; return TRUE;
} }
@ -551,16 +561,6 @@ gst_h264_decoder_preprocess_slice (GstH264Decoder * self, GstH264Slice * slice)
slice->header.first_mb_in_slice); slice->header.first_mb_in_slice);
return FALSE; return FALSE;
} }
/* If the new picture is an IDR, flush DPB */
if (slice->nalu.idr_pic_flag) {
/* Output all remaining pictures, unless we are explicitly instructed
* not to do so */
if (!slice->header.dec_ref_pic_marking.no_output_of_prior_pics_flag)
gst_h264_decoder_drain (GST_VIDEO_DECODER (self));
gst_h264_dpb_clear (priv->dpb);
}
} }
return TRUE; return TRUE;
@ -632,11 +632,29 @@ gst_h264_decoder_handle_frame_num_gap (GstH264Decoder * self, gint frame_num)
gst_h264_decoder_update_pic_nums (self, unused_short_term_frame_num); gst_h264_decoder_update_pic_nums (self, unused_short_term_frame_num);
if (!gst_h264_decoder_finish_picture (self, picture)) { /* C.2.1 */
GST_WARNING ("Failed to finish picture %p", picture); if (!gst_h264_decoder_sliding_window_picture_marking (self)) {
GST_ERROR_OBJECT (self,
"Couldn't perform sliding window picture marking");
return FALSE; return FALSE;
} }
gst_h264_dpb_delete_unused (priv->dpb);
gst_h264_dpb_add (priv->dpb, picture);
while (gst_h264_dpb_needs_bump (priv->dpb, priv->max_num_reorder_frames,
FALSE)) {
GstH264Picture *to_output;
to_output = gst_h264_dpb_bump (priv->dpb, FALSE);
if (!to_output) {
GST_WARNING_OBJECT (self, "Bumping is needed but no picture to output");
break;
}
gst_h264_decoder_do_output_picture (self, to_output);
}
unused_short_term_frame_num++; unused_short_term_frame_num++;
unused_short_term_frame_num %= priv->max_frame_num; unused_short_term_frame_num %= priv->max_frame_num;
} }
@ -677,6 +695,7 @@ gst_h264_decoder_start_current_picture (GstH264Decoder * self)
const GstH264SPS *sps; const GstH264SPS *sps;
gint frame_num; gint frame_num;
gboolean ret = TRUE; gboolean ret = TRUE;
GstH264Picture *current_picture;
g_assert (priv->current_picture != NULL); g_assert (priv->current_picture != NULL);
g_assert (priv->active_sps != NULL); g_assert (priv->active_sps != NULL);
@ -700,6 +719,25 @@ gst_h264_decoder_start_current_picture (GstH264Decoder * self)
if (!gst_h264_decoder_init_current_picture (self)) if (!gst_h264_decoder_init_current_picture (self))
return FALSE; return FALSE;
current_picture = priv->current_picture;
/* If the new picture is an IDR, flush DPB */
if (current_picture->idr) {
if (!current_picture->dec_ref_pic_marking.no_output_of_prior_pics_flag) {
gst_h264_decoder_drain_internal (self);
} else {
/* C.4.4 Removal of pictures from the DPB before possible insertion
* of the current picture
*
* If decoded picture is IDR and no_output_of_prior_pics_flag is equal to 1
* or is inferred to be equal to 1, all frame buffers in the DPB
* are emptied without output of the pictures they contain,
* and DPB fullness is set to 0.
*/
gst_h264_decoder_clear_dpb (self, FALSE);
}
}
gst_h264_decoder_update_pic_nums (self, frame_num); gst_h264_decoder_update_pic_nums (self, frame_num);
if (priv->process_ref_pic_lists) if (priv->process_ref_pic_lists)
@ -1207,24 +1245,12 @@ gst_h264_decoder_calculate_poc (GstH264Decoder * self, GstH264Picture * picture)
static void static void
gst_h264_decoder_do_output_picture (GstH264Decoder * self, gst_h264_decoder_do_output_picture (GstH264Decoder * self,
GstH264Picture * picture, gboolean clear_dpb) GstH264Picture * picture)
{ {
GstH264DecoderPrivate *priv = self->priv; GstH264DecoderPrivate *priv = self->priv;
GstH264DecoderClass *klass; GstH264DecoderClass *klass;
GstVideoCodecFrame *frame = NULL; GstVideoCodecFrame *frame = NULL;
picture->outputted = TRUE;
if (clear_dpb && !picture->ref)
gst_h264_dpb_delete_by_poc (priv->dpb, picture->pic_order_cnt);
if (picture->nonexisting) {
GST_DEBUG_OBJECT (self, "Skipping output, non-existing frame_num %d",
picture->frame_num);
gst_h264_picture_unref (picture);
return;
}
GST_LOG_OBJECT (self, "Outputting picture %p (frame_num %d, poc %d)", GST_LOG_OBJECT (self, "Outputting picture %p (frame_num %d, poc %d)",
picture, picture->frame_num, picture->pic_order_cnt); picture, picture->frame_num, picture->pic_order_cnt);
@ -1313,31 +1339,15 @@ static gboolean
gst_h264_decoder_drain_internal (GstH264Decoder * self) gst_h264_decoder_drain_internal (GstH264Decoder * self)
{ {
GstH264DecoderPrivate *priv = self->priv; GstH264DecoderPrivate *priv = self->priv;
GArray *to_output = priv->to_output; GstH264Picture *picture;
/* We are about to drain, so we can get rid of everything that has been while ((picture = gst_h264_dpb_bump (priv->dpb, TRUE)) != NULL) {
* outputted already */ gst_h264_decoder_do_output_picture (self, picture);
gst_h264_dpb_delete_outputted (priv->dpb);
gst_h264_dpb_get_pictures_not_outputted (priv->dpb, to_output);
g_array_sort (to_output, (GCompareFunc) poc_asc_compare);
while (to_output->len) {
GstH264Picture *picture = g_array_index (to_output, GstH264Picture *, 0);
/* We want the last reference when outputing so take a ref and then remove
* from both arrays. */
gst_h264_picture_ref (picture);
g_array_remove_index (to_output, 0);
gst_h264_dpb_delete_by_poc (priv->dpb, picture->pic_order_cnt);
GST_LOG_OBJECT (self, "Output picture %p (frame num %d, poc %d)", picture,
picture->frame_num, picture->pic_order_cnt);
gst_h264_decoder_do_output_picture (self, picture, FALSE);
} }
g_array_set_size (to_output, 0);
gst_h264_dpb_clear (priv->dpb); gst_h264_dpb_clear (priv->dpb);
priv->last_output_poc = 0; priv->last_output_poc = 0;
return TRUE; return TRUE;
} }
@ -1346,115 +1356,38 @@ gst_h264_decoder_handle_memory_management_opt (GstH264Decoder * self,
GstH264Picture * picture) GstH264Picture * picture)
{ {
GstH264DecoderPrivate *priv = self->priv; GstH264DecoderPrivate *priv = self->priv;
gint i, j; gint i;
for (i = 0; i < G_N_ELEMENTS (picture->dec_ref_pic_marking.ref_pic_marking); for (i = 0; i < G_N_ELEMENTS (picture->dec_ref_pic_marking.ref_pic_marking);
i++) { i++) {
GstH264RefPicMarking *ref_pic_marking = GstH264RefPicMarking *ref_pic_marking =
&picture->dec_ref_pic_marking.ref_pic_marking[i]; &picture->dec_ref_pic_marking.ref_pic_marking[i];
GstH264Picture *to_mark; guint8 type = ref_pic_marking->memory_management_control_operation;
gint pic_num_x;
switch (ref_pic_marking->memory_management_control_operation) { GST_TRACE_OBJECT (self, "memory management operation %d, type %d", i, type);
case 0:
/* Normal end of operations' specification */
return TRUE;
case 1:
/* Mark a short term reference picture as unused so it can be removed
* if outputted */
pic_num_x =
picture->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 +
1);
to_mark = gst_h264_dpb_get_short_ref_by_pic_num (priv->dpb, pic_num_x);
if (to_mark) {
to_mark->ref = FALSE;
} else {
GST_WARNING_OBJECT (self, "Invalid short term ref pic num to unmark");
return FALSE;
}
break;
case 2: /* Normal end of operations' specification */
/* Mark a long term reference picture as unused so it can be removed if (type == 0)
* if outputted */ return TRUE;
to_mark = gst_h264_dpb_get_long_ref_by_pic_num (priv->dpb,
ref_pic_marking->long_term_pic_num);
if (to_mark) {
to_mark->ref = FALSE;
} else {
GST_WARNING_OBJECT (self, "Invalid long term ref pic num to unmark");
return FALSE;
}
break;
case 3: switch (type) {
/* Mark a short term reference picture as long term reference */ case 4:
pic_num_x =
picture->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 +
1);
to_mark = gst_h264_dpb_get_short_ref_by_pic_num (priv->dpb, pic_num_x);
if (to_mark) {
to_mark->long_term = TRUE;
to_mark->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
} else {
GST_WARNING_OBJECT (self,
"Invalid short term ref pic num to mark as long ref");
return FALSE;
}
break;
case 4:{
GArray *pictures = gst_h264_dpb_get_pictures_all (priv->dpb);
/* Unmark all reference pictures with long_term_frame_idx over new max */
priv->max_long_term_frame_idx = priv->max_long_term_frame_idx =
ref_pic_marking->max_long_term_frame_idx_plus1 - 1; ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
for (j = 0; j < pictures->len; j++) {
GstH264Picture *pic = g_array_index (pictures, GstH264Picture *, j);
if (pic->long_term &&
pic->long_term_frame_idx > priv->max_long_term_frame_idx)
pic->ref = FALSE;
}
g_array_unref (pictures);
break; break;
}
case 5: case 5:
/* Unmark all reference pictures */
gst_h264_dpb_mark_all_non_ref (priv->dpb);
priv->max_long_term_frame_idx = -1; priv->max_long_term_frame_idx = -1;
picture->mem_mgmt_5 = TRUE;
break; break;
case 6:{
GArray *pictures = gst_h264_dpb_get_pictures_all (priv->dpb);
/* Replace long term reference pictures with current picture.
* First unmark if any existing with this long_term_frame_idx... */
for (j = 0; j < pictures->len; j++) {
GstH264Picture *pic = g_array_index (pictures, GstH264Picture *, j);
if (pic->long_term &&
pic->long_term_frame_idx == ref_pic_marking->long_term_frame_idx)
pic->ref = FALSE;
}
g_array_unref (pictures);
/* and mark the current one instead */
picture->ref = TRUE;
picture->long_term = TRUE;
picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
break;
}
default: default:
g_assert_not_reached ();
break; break;
} }
if (!gst_h264_dpb_perform_memory_management_control_operation (priv->dpb,
ref_pic_marking, picture)) {
GST_WARNING_OBJECT (self, "memory management operation type %d failed",
type);
return FALSE;
}
} }
return TRUE; return TRUE;
@ -1556,11 +1489,6 @@ gst_h264_decoder_finish_picture (GstH264Decoder * self,
GstH264Picture * picture) GstH264Picture * picture)
{ {
GstH264DecoderPrivate *priv = self->priv; GstH264DecoderPrivate *priv = self->priv;
GArray *not_outputted = priv->to_output;
guint num_remaining;
#ifndef GST_DISABLE_GST_DEBUG
gint i;
#endif
/* Finish processing the picture. /* Finish processing the picture.
* Start by storing previous picture data for later use */ * Start by storing previous picture data for later use */
@ -1581,184 +1509,26 @@ gst_h264_decoder_finish_picture (GstH264Decoder * self,
/* Remove unused (for reference or later output) pictures from DPB, marking /* Remove unused (for reference or later output) pictures from DPB, marking
* them as such */ * them as such */
gst_h264_dpb_delete_unused (priv->dpb); gst_h264_dpb_delete_unused (priv->dpb);
gst_h264_dpb_add (priv->dpb, picture);
GST_LOG_OBJECT (self, GST_LOG_OBJECT (self,
"Finishing picture %p (frame_num %d, poc %d), entries in DPB %d", "Finishing picture %p (frame_num %d, poc %d), entries in DPB %d",
picture, picture->frame_num, picture->pic_order_cnt, picture, picture->frame_num, picture->pic_order_cnt,
gst_h264_dpb_get_size (priv->dpb)); gst_h264_dpb_get_size (priv->dpb));
/* The ownership of pic will either be transferred to DPB - if the picture is while (gst_h264_dpb_needs_bump (priv->dpb, priv->max_num_reorder_frames,
* still needed (for output and/or reference) - or we will release it priv->is_live)) {
* immediately if we manage to output it here and won't have to store it for GstH264Picture *to_output;
* future reference */
/* Get all pictures that haven't been outputted yet */ to_output = gst_h264_dpb_bump (priv->dpb, FALSE);
gst_h264_dpb_get_pictures_not_outputted (priv->dpb, not_outputted);
/* Include the one we've just decoded */
g_array_append_val (not_outputted, picture);
/* for debugging */ if (!to_output) {
#ifndef GST_DISABLE_GST_DEBUG GST_WARNING_OBJECT (self, "Bumping is needed but no picture to output");
if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE) { break;
GST_TRACE_OBJECT (self, "Before sorting not outputted list");
for (i = 0; i < not_outputted->len; i++) {
GstH264Picture *tmp = g_array_index (not_outputted, GstH264Picture *, i);
GST_TRACE_OBJECT (self,
"\t%dth picture %p (frame_num %d, poc %d)", i, tmp,
tmp->frame_num, tmp->pic_order_cnt);
} }
gst_h264_decoder_do_output_picture (self, to_output);
} }
#endif
/* Sort in output order */
g_array_sort (not_outputted, (GCompareFunc) poc_asc_compare);
#ifndef GST_DISABLE_GST_DEBUG
if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE) {
GST_TRACE_OBJECT (self,
"After sorting not outputted list in poc ascending order");
for (i = 0; i < not_outputted->len; i++) {
GstH264Picture *tmp = g_array_index (not_outputted, GstH264Picture *, i);
GST_TRACE_OBJECT (self,
"\t%dth picture %p (frame_num %d, poc %d)", i, tmp,
tmp->frame_num, tmp->pic_order_cnt);
}
}
#endif
/* Try to output as many pictures as we can. A picture can be output,
* if the number of decoded and not yet outputted pictures that would remain
* in DPB afterwards would at least be equal to max_num_reorder_frames.
* If the outputted picture is not a reference picture, it doesn't have
* to remain in the DPB and can be removed */
num_remaining = not_outputted->len;
while (num_remaining > priv->max_num_reorder_frames ||
/* If the condition below is used, this is an invalid stream. We should
* not be forced to output beyond max_num_reorder_frames in order to
* make room in DPB to store the current picture (if we need to do so).
* However, if this happens, ignore max_num_reorder_frames and try
* to output more. This may cause out-of-order output, but is not
* fatal, and better than failing instead */
((gst_h264_dpb_is_full (priv->dpb) && (picture && (!picture->outputted
|| picture->ref)))
&& num_remaining)) {
gboolean clear_dpb = TRUE;
GstH264Picture *to_output =
g_array_index (not_outputted, GstH264Picture *, 0);
gst_h264_picture_ref (to_output);
g_array_remove_index (not_outputted, 0);
if (num_remaining <= priv->max_num_reorder_frames) {
GST_WARNING_OBJECT (self,
"Invalid stream, max_num_reorder_frames not preserved");
}
GST_LOG_OBJECT (self,
"Output picture %p (frame num %d)", to_output, to_output->frame_num);
/* Current picture hasn't been inserted into DPB yet, so don't remove it
* if we managed to output it immediately */
if (picture && to_output == picture) {
clear_dpb = FALSE;
if (picture->ref) {
GST_TRACE_OBJECT (self,
"Put current picture %p (frame num %d, poc %d) to dpb",
picture, picture->frame_num, picture->pic_order_cnt);
gst_h264_dpb_add (priv->dpb, gst_h264_picture_ref (picture));
}
/* and mark current picture as handled */
picture = NULL;
}
gst_h264_decoder_do_output_picture (self, to_output, clear_dpb);
num_remaining--;
}
/* If we haven't managed to output the picture that we just decoded, or if
* it's a reference picture, we have to store it in DPB */
if (picture && (!picture->outputted || picture->ref)) {
if (gst_h264_dpb_is_full (priv->dpb)) {
/* If we haven't managed to output anything to free up space in DPB
* to store this picture, it's an error in the stream */
GST_WARNING_OBJECT (self, "Could not free up space in DPB");
g_array_set_size (not_outputted, 0);
return FALSE;
}
GST_TRACE_OBJECT (self,
"Put picture %p (outputted %d, ref %d, frame num %d, poc %d) to dpb",
picture, picture->outputted, picture->ref, picture->frame_num,
picture->pic_order_cnt);
gst_h264_dpb_add (priv->dpb, gst_h264_picture_ref (picture));
}
/* clear possible reference to the current picture.
* If *picture* is still non-null, it means that the current picture not
* outputted yet, and DPB may or may not hold the reference of the picture */
if (picture)
gst_h264_picture_ref (picture);
g_array_set_size (not_outputted, 0);
/* C.4.5.3 "Bumping" process for non-DPB full case, DPB full cases should be
* covered above */
/* FIXME: should cover interlaced streams */
if (picture && !picture->outputted &&
picture->field == GST_H264_PICTURE_FIELD_FRAME) {
gboolean do_output = TRUE;
if (picture->idr &&
!picture->dec_ref_pic_marking.no_output_of_prior_pics_flag) {
/* The current picture is an IDR picture and no_output_of_prior_pics_flag
* is not equal to 1 and is not inferred to be equal to 1, as specified
* in clause C.4.4 */
GST_TRACE_OBJECT (self, "Output IDR picture");
} else if (picture->mem_mgmt_5) {
/* The current picture has memory_management_control_operation equal to 5,
* as specified in clause C.4.4 */
GST_TRACE_OBJECT (self, "Output mem_mgmt_5 picture");
} else if (priv->last_output_poc >= 0 &&
picture->pic_order_cnt > priv->last_output_poc &&
(picture->pic_order_cnt - priv->last_output_poc) <= 2 &&
/* NOTE: this might have a negative effect on throughput performance
* depending on hardware implementation.
* TODO: Possible solution is threading but it would make decoding flow
* very complicated. */
priv->is_live) {
/* NOTE: this condition is not specified by spec but we can output
* this picture based on calculated POC and last outputted POC */
/* NOTE: The assumption here is, every POC of frame will have step of two.
* however, if the assumption is wrong, (i.e., POC step is one, not two),
* this would break output order. If this assumption is wrong,
* please remove this condition.
*/
GST_LOG_OBJECT (self,
"Forcing output picture %p (frame num %d, poc %d, last poc %d)",
picture, picture->frame_num, picture->pic_order_cnt,
priv->last_output_poc);
} else {
do_output = FALSE;
GST_TRACE_OBJECT (self, "Current picture %p (frame num %d, poc %d) "
"is not ready to be output picture",
picture, picture->frame_num, picture->pic_order_cnt);
}
if (do_output) {
/* pass ownership of the current picture. At this point,
* dpb must be holding a reference of the current picture */
gst_h264_decoder_do_output_picture (self, picture, TRUE);
picture = NULL;
}
}
if (picture)
gst_h264_picture_unref (picture);
return TRUE; return TRUE;
} }

449
gst-libs/gst/codecs/gsth264picture.c
View file

@ -22,6 +22,7 @@
#endif #endif
#include "gsth264picture.h" #include "gsth264picture.h"
#include <stdlib.h>
GST_DEBUG_CATEGORY_EXTERN (gst_h264_decoder_debug); GST_DEBUG_CATEGORY_EXTERN (gst_h264_decoder_debug);
#define GST_CAT_DEFAULT gst_h264_decoder_debug #define GST_CAT_DEFAULT gst_h264_decoder_debug
@ -106,8 +107,17 @@ struct _GstH264Dpb
{ {
GArray *pic_list; GArray *pic_list;
gint max_num_pics; gint max_num_pics;
gint num_output_needed;
gint32 last_output_poc;
}; };
static void
gst_h264_dpb_init (GstH264Dpb * dpb)
{
dpb->num_output_needed = 0;
dpb->last_output_poc = G_MININT32;
}
/** /**
* gst_h264_dpb_new: (skip) * gst_h264_dpb_new: (skip)
* *
@ -121,6 +131,7 @@ gst_h264_dpb_new (void)
GstH264Dpb *dpb; GstH264Dpb *dpb;
dpb = g_new0 (GstH264Dpb, 1); dpb = g_new0 (GstH264Dpb, 1);
gst_h264_dpb_init (dpb);
dpb->pic_list = dpb->pic_list =
g_array_sized_new (FALSE, TRUE, sizeof (GstH264Picture *), g_array_sized_new (FALSE, TRUE, sizeof (GstH264Picture *),
@ -188,6 +199,7 @@ gst_h264_dpb_clear (GstH264Dpb * dpb)
g_return_if_fail (dpb != NULL); g_return_if_fail (dpb != NULL);
g_array_set_size (dpb->pic_list, 0); g_array_set_size (dpb->pic_list, 0);
gst_h264_dpb_init (dpb);
} }
/** /**
@ -203,6 +215,17 @@ gst_h264_dpb_add (GstH264Dpb * dpb, GstH264Picture * picture)
g_return_if_fail (dpb != NULL); g_return_if_fail (dpb != NULL);
g_return_if_fail (GST_IS_H264_PICTURE (picture)); g_return_if_fail (GST_IS_H264_PICTURE (picture));
/* C.4.2 Decoding of gaps in frame_num and storage of "non-existing" pictures
*
* The "non-existing" frame is stored in an empty frame buffer and is marked
* as "not needed for output", and the DPB fullness is incremented by one */
if (!picture->nonexisting) {
picture->needed_for_output = TRUE;
dpb->num_output_needed++;
} else {
picture->needed_for_output = FALSE;
}
g_array_append_val (dpb->pic_list, picture); g_array_append_val (dpb->pic_list, picture);
} }
@ -223,73 +246,17 @@ gst_h264_dpb_delete_unused (GstH264Dpb * dpb)
GstH264Picture *picture = GstH264Picture *picture =
g_array_index (dpb->pic_list, GstH264Picture *, i); g_array_index (dpb->pic_list, GstH264Picture *, i);
if (picture->outputted && !picture->ref) { /* NOTE: don't use g_array_remove_index_fast here since the last picture
* need to be referenced for bumping decision */
if (!picture->needed_for_output && !picture->ref) {
GST_TRACE ("remove picture %p (frame num %d) from dpb", GST_TRACE ("remove picture %p (frame num %d) from dpb",
picture, picture->frame_num); picture, picture->frame_num);
g_array_remove_index_fast (dpb->pic_list, i); g_array_remove_index (dpb->pic_list, i);
i--; i--;
} }
} }
} }
/**
* gst_h264_dpb_delete_outputted:
* @dpb: a #GstH264Dpb
*
* Delete already outputted picture, even if they are referenced.
*
* Since: 1.18
*/
void
gst_h264_dpb_delete_outputted (GstH264Dpb * dpb)
{
gint i;
g_return_if_fail (dpb != NULL);
for (i = 0; i < dpb->pic_list->len; i++) {
GstH264Picture *picture =
g_array_index (dpb->pic_list, GstH264Picture *, i);
if (picture->outputted) {
GST_TRACE ("remove picture %p (frame num %d) from dpb",
picture, picture->frame_num);
g_array_remove_index_fast (dpb->pic_list, i);
i--;
}
}
}
/**
* gst_h264_dpb_delete_by_poc:
* @dpb: a #GstH264Dpb
* @poc: a poc of #GstH264Picture to remove
*
* Delete a #GstH264Dpb by @poc
*/
void
gst_h264_dpb_delete_by_poc (GstH264Dpb * dpb, gint poc)
{
gint i;
g_return_if_fail (dpb != NULL);
for (i = 0; i < dpb->pic_list->len; i++) {
GstH264Picture *picture =
g_array_index (dpb->pic_list, GstH264Picture *, i);
if (picture->pic_order_cnt == poc) {
GST_TRACE ("remove picture %p for poc %d (frame num %d) from dpb",
picture, poc, picture->frame_num);
g_array_remove_index_fast (dpb->pic_list, i);
return;
}
}
GST_WARNING ("Couldn't find picture with poc %d", poc);
}
/** /**
* gst_h264_dpb_num_ref_pictures: * gst_h264_dpb_num_ref_pictures:
* @dpb: a #GstH264Dpb * @dpb: a #GstH264Dpb
@ -425,33 +392,6 @@ gst_h264_dpb_get_lowest_frame_num_short_ref (GstH264Dpb * dpb)
return ret; return ret;
} }
/**
* gst_h264_dpb_get_pictures_not_outputted:
* @dpb: a #GstH264Dpb
* @out: (out) (element-type GstH264Picture) (transfer full): an array
* of #GstH264Picture pointer
*
* Retrieve all not-outputted pictures from @dpb
*/
void
gst_h264_dpb_get_pictures_not_outputted (GstH264Dpb * dpb, GArray * out)
{
gint i;
g_return_if_fail (dpb != NULL);
g_return_if_fail (out != NULL);
for (i = 0; i < dpb->pic_list->len; i++) {
GstH264Picture *picture =
g_array_index (dpb->pic_list, GstH264Picture *, i);
if (!picture->outputted) {
gst_h264_picture_ref (picture);
g_array_append_val (out, picture);
}
}
}
/** /**
* gst_h264_dpb_get_pictures_short_term_ref: * gst_h264_dpb_get_pictures_short_term_ref:
* @dpb: a #GstH264Dpb * @dpb: a #GstH264Dpb
@ -537,20 +477,6 @@ gst_h264_dpb_get_size (GstH264Dpb * dpb)
return dpb->pic_list->len; return dpb->pic_list->len;
} }
/**
* gst_h264_dpb_is_full:
* @dpb: a #GstH264Dpb
*
* Return: %TRUE if @dpb is full
*/
gboolean
gst_h264_dpb_is_full (GstH264Dpb * dpb)
{
g_return_val_if_fail (dpb != NULL, -1);
return dpb->pic_list->len >= dpb->max_num_pics;
}
/** /**
* gst_h264_dpb_get_picture: * gst_h264_dpb_get_picture:
* @dpb: a #GstH264Dpb * @dpb: a #GstH264Dpb
@ -581,3 +507,324 @@ gst_h264_dpb_get_picture (GstH264Dpb * dpb, guint32 system_frame_number)
return NULL; return NULL;
} }
static gboolean
gst_h264_dpb_has_empty_frame_buffer (GstH264Dpb * dpb)
{
if (dpb->pic_list->len <= dpb->max_num_pics)
return TRUE;
return FALSE;
}
static gint
gst_h264_dpb_get_lowest_output_needed_picture (GstH264Dpb * dpb,
GstH264Picture ** picture)
{
gint i;
GstH264Picture *lowest = NULL;
gint index = -1;
*picture = NULL;
for (i = 0; i < dpb->pic_list->len; i++) {
GstH264Picture *picture =
g_array_index (dpb->pic_list, GstH264Picture *, i);
if (!picture->needed_for_output)
continue;
if (!lowest) {
lowest = picture;
index = i;
continue;
}
if (picture->pic_order_cnt < lowest->pic_order_cnt) {
lowest = picture;
index = i;
}
}
if (lowest)
*picture = gst_h264_picture_ref (lowest);
return index;
}
/**
* gst_h264_dpb_needs_bump:
* @dpb: a #GstH264Dpb
* @current_picture: a #GstH264Picture currently decoded but not added to dpb
* @low_latency: %TRUE if low-latency bumping is required
*
* Returns: %TRUE if bumping is required
*
* Since: 1.20
*/
gboolean
gst_h264_dpb_needs_bump (GstH264Dpb * dpb, guint32 max_num_reorder_frames,
gboolean low_latency)
{
GstH264Picture *current_picture;
g_return_val_if_fail (dpb != NULL, FALSE);
g_assert (dpb->num_output_needed >= 0);
/* Empty so nothing to bump */
if (dpb->pic_list->len == 0 || dpb->num_output_needed == 0)
return FALSE;
/* FIXME: Need to revisit for intelaced decoding */
/* Case 1)
* C.4.2 Decoding of gaps in frame_num and storage of "non-existing" pictures
* C.4.5.1 Storage and marking of a reference decoded picture into the DPB
* C.4.5.2 Storage and marking of a non-reference decoded picture into the DPB
*
* In summary, if DPB is full and there is no empty space to store current
* picture, need bumping.
* NOTE: current picture was added already by our decoding flow, So we need to
* do bumping until dpb->pic_list->len == dpb->max_num_pic
*/
if (!gst_h264_dpb_has_empty_frame_buffer (dpb)) {
GST_TRACE ("No empty frame buffer, need bumping");
return TRUE;
}
if (dpb->num_output_needed > max_num_reorder_frames) {
GST_TRACE
("not outputted frames (%d) > max_num_reorder_frames (%d), need bumping",
dpb->num_output_needed, max_num_reorder_frames);
return TRUE;
}
current_picture =
g_array_index (dpb->pic_list, GstH264Picture *, dpb->pic_list->len - 1);
if (current_picture->needed_for_output && current_picture->idr &&
!current_picture->dec_ref_pic_marking.no_output_of_prior_pics_flag) {
GST_TRACE ("IDR with no_output_of_prior_pics_flag == 0, need bumping");
return TRUE;
}
if (current_picture->needed_for_output && current_picture->mem_mgmt_5) {
GST_TRACE ("Memory management type 5, need bumping");
return TRUE;
}
/* HACK: Not all streams have PicOrderCnt increment by 2, but in practice this
* condition can be used */
if (low_latency && dpb->last_output_poc != G_MININT32) {
GstH264Picture *picture = NULL;
gint32 lowest_poc = G_MININT32;
gst_h264_dpb_get_lowest_output_needed_picture (dpb, &picture);
if (picture) {
lowest_poc = picture->pic_order_cnt;
gst_h264_picture_unref (picture);
}
if (lowest_poc != G_MININT32 && lowest_poc > dpb->last_output_poc
&& abs (lowest_poc - dpb->last_output_poc) <= 2) {
GST_TRACE ("bumping for low-latency, lowest-poc: %d, last-output-poc: %d",
lowest_poc, dpb->last_output_poc);
return TRUE;
}
}
return FALSE;
}
/**
* gst_h264_dpb_bump:
* @dpb: a #GstH265Dpb
* @drain: whether draining or not
*
* Perform bumping process as defined in C.4.5.3 "Bumping" process.
* If @drain is %TRUE, @dpb will remove a #GstH264Picture from internal array
* so that returned #GstH264Picture could hold the last reference of it
*
* Returns: (nullable) (transfer full): a #GstH264Picture which is needed to be
* outputted
*
* Since: 1.20
*/
GstH264Picture *
gst_h264_dpb_bump (GstH264Dpb * dpb, gboolean drain)
{
GstH264Picture *picture;
gint index;
g_return_val_if_fail (dpb != NULL, NULL);
index = gst_h264_dpb_get_lowest_output_needed_picture (dpb, &picture);
if (!picture || index < 0)
return NULL;
picture->needed_for_output = FALSE;
dpb->num_output_needed--;
g_assert (dpb->num_output_needed >= 0);
/* NOTE: don't use g_array_remove_index_fast here since the last picture
* need to be referenced for bumping decision */
if (!picture->ref)
g_array_remove_index (dpb->pic_list, index);
dpb->last_output_poc = picture->pic_order_cnt;
return picture;
}
static gint
get_picNumX (GstH264Picture * picture, GstH264RefPicMarking * ref_pic_marking)
{
/* FIXME: support interlaced */
return picture->pic_num -
(ref_pic_marking->difference_of_pic_nums_minus1 + 1);
}
/**
* gst_h264_dpb_perform_memory_management_control_operation:
* @dpb: a #GstH265Dpb
* @ref_pic_marking: a #GstH264RefPicMarking
* @picture: a #GstH264Picture
*
* Perform "8.2.5.4 Adaptive memory control decoded reference picture marking process"
*
* Returns: %TRUE if successful
*
* Since: 1.20
*/
gboolean
gst_h264_dpb_perform_memory_management_control_operation (GstH264Dpb * dpb,
GstH264RefPicMarking * ref_pic_marking, GstH264Picture * picture)
{
guint8 type;
gint pic_num_x;
gint max_long_term_frame_idx;
GstH264Picture *other;
gint i;
g_return_val_if_fail (dpb != NULL, FALSE);
g_return_val_if_fail (ref_pic_marking != NULL, FALSE);
g_return_val_if_fail (picture != NULL, FALSE);
type = ref_pic_marking->memory_management_control_operation;
switch (type) {
case 0:
/* Normal end of operations' specification */
break;
case 1:
/* 8.2.5.4.1 Mark a short term reference picture as unused so it can be
* removed if outputted */
pic_num_x = get_picNumX (picture, ref_pic_marking);
other = gst_h264_dpb_get_short_ref_by_pic_num (dpb, pic_num_x);
if (other) {
other->ref = FALSE;
} else {
GST_WARNING ("Invalid picNumX %d for operation type 1", pic_num_x);
return FALSE;
}
break;
case 2:
/* 8.2.5.4.2 Mark a long term reference picture as unused so it can be
* removed if outputted */
other = gst_h264_dpb_get_long_ref_by_pic_num (dpb,
ref_pic_marking->long_term_pic_num);
if (other) {
other->ref = FALSE;
} else {
GST_WARNING ("Invalid LongTermPicNum %d for operation type 2",
ref_pic_marking->long_term_pic_num);
return FALSE;
}
break;
case 3:
/* 8.2.5.4.3 Mark a short term reference picture as long term reference */
/* If we have long-term ref picture for LongTermFrameIdx,
* mark the picture as non-reference */
for (i = 0; i < dpb->pic_list->len; i++) {
other = g_array_index (dpb->pic_list, GstH264Picture *, i);
if (other->ref && other->long_term && other->long_term_frame_idx ==
ref_pic_marking->long_term_frame_idx) {
GST_LOG ("Unmark old long-term ref pic %p (poc %d)",
other, other->pic_order_cnt);
other->ref = FALSE;
other->long_term = FALSE;
break;
}
}
pic_num_x = get_picNumX (picture, ref_pic_marking);
other = gst_h264_dpb_get_short_ref_by_pic_num (dpb, pic_num_x);
if (other) {
other->long_term = TRUE;;
other->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
} else {
GST_WARNING ("Invalid picNumX %d for operation type 3", pic_num_x);
return FALSE;
}
break;
case 4:
/* 8.2.5.4.4 All pictures for which LongTermFrameIdx is greater than
* max_long_term_frame_idx_plus1 − 1 and that are marked as
* "used for long-term reference" are marked as "unused for reference */
max_long_term_frame_idx =
ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
for (i = 0; i < dpb->pic_list->len; i++) {
other = g_array_index (dpb->pic_list, GstH264Picture *, i);
if (other->ref && other->long_term &&
other->long_term_frame_idx > max_long_term_frame_idx) {
other->ref = FALSE;
other->long_term = FALSE;
}
}
break;
case 5:
/* 8.2.5.4.5 Unmark all reference pictures */
for (i = 0; i < dpb->pic_list->len; i++) {
other = g_array_index (dpb->pic_list, GstH264Picture *, i);
other->ref = FALSE;
other->long_term = FALSE;
}
picture->mem_mgmt_5 = TRUE;
break;
case 6:
/* 8.2.5.4.6 Replace long term reference pictures with current picture.
* First unmark if any existing with this long_term_frame_idx */
/* If we have long-term ref picture for LongTermFrameIdx,
* mark the picture as non-reference */
for (i = 0; i < dpb->pic_list->len; i++) {
other = g_array_index (dpb->pic_list, GstH264Picture *, i);
if (other->ref && other->long_term && other->long_term_frame_idx ==
ref_pic_marking->long_term_frame_idx) {
GST_LOG ("Unmark old long-term ref pic %p (poc %d)",
other, other->pic_order_cnt);
other->ref = FALSE;
other->long_term = FALSE;
break;
}
}
picture->ref = TRUE;
picture->long_term = TRUE;
picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
break;
default:
g_assert_not_reached ();
return FALSE;
}
return TRUE;
}

27
gst-libs/gst/codecs/gsth264picture.h
View file

@ -54,6 +54,7 @@ typedef enum
struct _GstH264Picture struct _GstH264Picture
{ {
/*< private >*/
GstMiniObject parent; GstMiniObject parent;
GstH264SliceType type; GstH264SliceType type;
@ -84,7 +85,7 @@ struct _GstH264Picture
gint idr_pic_id; gint idr_pic_id;
gboolean ref; gboolean ref;
gboolean long_term; gboolean long_term;
gboolean outputted; gboolean needed_for_output;
gboolean mem_mgmt_5; gboolean mem_mgmt_5;
gboolean nonexisting; gboolean nonexisting;
@ -168,13 +169,6 @@ void gst_h264_dpb_add (GstH264Dpb * dpb,
GST_CODECS_API GST_CODECS_API
void gst_h264_dpb_delete_unused (GstH264Dpb * dpb); void gst_h264_dpb_delete_unused (GstH264Dpb * dpb);
GST_CODECS_API
void gst_h264_dpb_delete_outputted (GstH264Dpb * dpb);
GST_CODECS_API
void gst_h264_dpb_delete_by_poc (GstH264Dpb * dpb,
gint poc);
GST_CODECS_API GST_CODECS_API
gint gst_h264_dpb_num_ref_pictures (GstH264Dpb * dpb); gint gst_h264_dpb_num_ref_pictures (GstH264Dpb * dpb);
@ -192,10 +186,6 @@ GstH264Picture * gst_h264_dpb_get_long_ref_by_pic_num (GstH264Dpb * dpb,
GST_CODECS_API GST_CODECS_API
GstH264Picture * gst_h264_dpb_get_lowest_frame_num_short_ref (GstH264Dpb * dpb); GstH264Picture * gst_h264_dpb_get_lowest_frame_num_short_ref (GstH264Dpb * dpb);
GST_CODECS_API
void gst_h264_dpb_get_pictures_not_outputted (GstH264Dpb * dpb,
GArray * out);
GST_CODECS_API GST_CODECS_API
void gst_h264_dpb_get_pictures_short_term_ref (GstH264Dpb * dpb, void gst_h264_dpb_get_pictures_short_term_ref (GstH264Dpb * dpb,
GArray * out); GArray * out);
@ -215,7 +205,18 @@ GST_CODECS_API
gint gst_h264_dpb_get_size (GstH264Dpb * dpb); gint gst_h264_dpb_get_size (GstH264Dpb * dpb);
GST_CODECS_API GST_CODECS_API
gboolean gst_h264_dpb_is_full (GstH264Dpb * dpb); gboolean gst_h264_dpb_needs_bump (GstH264Dpb * dpb,
guint32 max_num_reorder_frames,
gboolean low_latency);
GST_CODECS_API
GstH264Picture * gst_h264_dpb_bump (GstH264Dpb * dpb,
gboolean drain);
GST_CODECS_API
gboolean gst_h264_dpb_perform_memory_management_control_operation (GstH264Dpb * dpb,
GstH264RefPicMarking *ref_pic_marking,
GstH264Picture * picture);
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstH264Picture, gst_h264_picture_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstH264Picture, gst_h264_picture_unref)