gstreamer/gst-libs/gst/codecs/gsth265picture.c
He Junyan c442c9bd5e codecs: Integrate H265 DPB full check into need_bump().
The current DPB check of H265 is not very correct. The current frame
is already in the DPB when we check whether the DPB is full.
For example, the DPB max size is 16 and we have 15 ref frames in the
DPB, so the gst_h265_dpb_delete_unused() cleans no one, and then plus
the current frame, the DPB is 16. This causes an error return, but in
fact, the stream is correct.
We now integrate the DPB full check into the need_bump() function.
We add the correct frame into to DPB and then check whether the picture
num is bigger than max_num_pics of DPB(which means there is no room for
the current picture). If true, we bump the DPB immediately.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2291>
2021-06-02 13:10:35 +00:00

628 lines
14 KiB
C

/* GStreamer
* Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include "gsth265picture.h"
GST_DEBUG_CATEGORY_EXTERN (gst_h265_decoder_debug);
#define GST_CAT_DEFAULT gst_h265_decoder_debug
GST_DEFINE_MINI_OBJECT_TYPE (GstH265Picture, gst_h265_picture);
static void
_gst_h265_picture_free (GstH265Picture * picture)
{
if (picture->notify)
picture->notify (picture->user_data);
g_free (picture);
}
/**
* gst_h265_picture_new:
*
* Create new #GstH265Picture
*
* Returns: a new #GstH265Picture
*/
GstH265Picture *
gst_h265_picture_new (void)
{
GstH265Picture *pic;
pic = g_new0 (GstH265Picture, 1);
pic->pts = GST_CLOCK_TIME_NONE;
pic->pic_struct = GST_H265_SEI_PIC_STRUCT_FRAME;
/* 0: interlaced, 1: progressive, 2: unspecified, 3: reserved, can be
* interpreted as 2 */
pic->source_scan_type = 2;
pic->duplicate_flag = 0;
gst_mini_object_init (GST_MINI_OBJECT_CAST (pic), 0,
GST_TYPE_H265_PICTURE, NULL, NULL,
(GstMiniObjectFreeFunction) _gst_h265_picture_free);
return pic;
}
/**
* gst_h265_picture_set_user_data:
* @picture: a #GstH265Picture
* @user_data: private data
* @notify: (closure user_data): a #GDestroyNotify
*
* Sets @user_data on the picture and the #GDestroyNotify that will be called when
* the picture is freed.
*
* If a @user_data was previously set, then the previous set @notify will be called
* before the @user_data is replaced.
*/
void
gst_h265_picture_set_user_data (GstH265Picture * picture, gpointer user_data,
GDestroyNotify notify)
{
g_return_if_fail (GST_IS_H265_PICTURE (picture));
if (picture->notify)
picture->notify (picture->user_data);
picture->user_data = user_data;
picture->notify = notify;
}
/**
* gst_h265_picture_get_user_data:
* @picture: a #GstH265Picture
*
* Gets private data set on the picture via
* gst_h265_picture_set_user_data() previously.
*
* Returns: (transfer none): The previously set user_data
*/
gpointer
gst_h265_picture_get_user_data (GstH265Picture * picture)
{
return picture->user_data;
}
struct _GstH265Dpb
{
GArray *pic_list;
gint max_num_pics;
gint num_output_needed;
};
/**
* gst_h265_dpb_new: (skip)
*
* Create new #GstH265Dpb
*
* Returns: a new #GstH265Dpb
*/
GstH265Dpb *
gst_h265_dpb_new (void)
{
GstH265Dpb *dpb;
dpb = g_new0 (GstH265Dpb, 1);
dpb->pic_list =
g_array_sized_new (FALSE, TRUE, sizeof (GstH265Picture *),
GST_H265_DPB_MAX_SIZE);
g_array_set_clear_func (dpb->pic_list,
(GDestroyNotify) gst_h265_picture_clear);
return dpb;
}
/**
* gst_h265_dpb_set_max_num_pics:
* @dpb: a #GstH265Dpb
* @max_num_pics: the maximum number of picture
*
* Set the number of maximum allowed pictures to store
*/
void
gst_h265_dpb_set_max_num_pics (GstH265Dpb * dpb, gint max_num_pics)
{
g_return_if_fail (dpb != NULL);
dpb->max_num_pics = max_num_pics;
}
/**
* gst_h265_dpb_get_max_num_pics:
* @dpb: a #GstH265Dpb
*
* Returns: the number of maximum pictures
*/
gint
gst_h265_dpb_get_max_num_pics (GstH265Dpb * dpb)
{
g_return_val_if_fail (dpb != NULL, 0);
return dpb->max_num_pics;
}
/**
* gst_h265_dpb_free:
* @dpb: a #GstH265Dpb to free
*
* Free the @dpb
*/
void
gst_h265_dpb_free (GstH265Dpb * dpb)
{
g_return_if_fail (dpb != NULL);
gst_h265_dpb_clear (dpb);
g_array_unref (dpb->pic_list);
g_free (dpb);
}
/**
* gst_h265_dpb_clear:
* @dpb: a #GstH265Dpb
*
* Clear all stored #GstH265Picture
*/
void
gst_h265_dpb_clear (GstH265Dpb * dpb)
{
g_return_if_fail (dpb != NULL);
g_array_set_size (dpb->pic_list, 0);
dpb->num_output_needed = 0;
}
/**
* gst_h265_dpb_add:
* @dpb: a #GstH265Dpb
* @picture: (transfer full): a #GstH265Picture
*
* Store the @picture and perform increase pic_latency_cnt as defined in
* "C.5.2.3 Additional bumping" process
*/
void
gst_h265_dpb_add (GstH265Dpb * dpb, GstH265Picture * picture)
{
g_return_if_fail (dpb != NULL);
g_return_if_fail (GST_IS_H265_PICTURE (picture));
if (picture->output_flag) {
gint i;
for (i = 0; i < dpb->pic_list->len; i++) {
GstH265Picture *other =
g_array_index (dpb->pic_list, GstH265Picture *, i);
if (other->needed_for_output)
other->pic_latency_cnt++;
}
dpb->num_output_needed++;
picture->needed_for_output = TRUE;
} else {
picture->needed_for_output = FALSE;
}
/* C.3.4 */
picture->ref = TRUE;
picture->long_term = FALSE;
g_array_append_val (dpb->pic_list, picture);
}
/**
* gst_h265_dpb_delete_unused:
* @dpb: a #GstH265Dpb
*
* Delete not needed for output and not referenced all pictures from dpb
*/
void
gst_h265_dpb_delete_unused (GstH265Dpb * dpb)
{
gint i;
g_return_if_fail (dpb != NULL);
for (i = 0; i < dpb->pic_list->len; i++) {
GstH265Picture *picture =
g_array_index (dpb->pic_list, GstH265Picture *, i);
if (!picture->needed_for_output && !picture->ref) {
GST_TRACE ("remove picture %p (poc %d) from dpb",
picture, picture->pic_order_cnt);
g_array_remove_index (dpb->pic_list, i);
i--;
}
}
}
/**
* gst_h265_dpb_num_ref_pictures:
* @dpb: a #GstH265Dpb
*
* Returns: The number of referenced pictures
*/
gint
gst_h265_dpb_num_ref_pictures (GstH265Dpb * dpb)
{
gint i;
gint ret = 0;
g_return_val_if_fail (dpb != NULL, -1);
for (i = 0; i < dpb->pic_list->len; i++) {
GstH265Picture *picture =
g_array_index (dpb->pic_list, GstH265Picture *, i);
if (picture->ref)
ret++;
}
return ret;
}
/**
* gst_h265_dpb_mark_all_non_ref:
* @dpb: a #GstH265Dpb
*
* Mark all pictures are not referenced
*/
void
gst_h265_dpb_mark_all_non_ref (GstH265Dpb * dpb)
{
gint i;
g_return_if_fail (dpb != NULL);
for (i = 0; i < dpb->pic_list->len; i++) {
GstH265Picture *picture =
g_array_index (dpb->pic_list, GstH265Picture *, i);
picture->ref = FALSE;
}
}
/**
* gst_h265_dpb_get_ref_by_poc:
* @dpb: a #GstH265Dpb
* @poc: a picture order count
*
* Find a short or long term reference picture which has matching poc
*
* Returns: (nullable) (transfer full): a #GstH265Picture
*/
GstH265Picture *
gst_h265_dpb_get_ref_by_poc (GstH265Dpb * dpb, gint poc)
{
gint i;
g_return_val_if_fail (dpb != NULL, NULL);
for (i = 0; i < dpb->pic_list->len; i++) {
GstH265Picture *picture =
g_array_index (dpb->pic_list, GstH265Picture *, i);
if (picture->ref && picture->pic_order_cnt == poc)
return gst_h265_picture_ref (picture);
}
GST_DEBUG ("No short term reference picture for %d", poc);
return NULL;
}
/**
* gst_h265_dpb_get_ref_by_poc_lsb:
* @dpb: a #GstH265Dpb
* @poc_lsb: a picture order count lsb
*
* Find a short or long term reference picture which has matching poc_lsb
*
* Returns: (nullable) (transfer full): a #GstH265Picture
*/
GstH265Picture *
gst_h265_dpb_get_ref_by_poc_lsb (GstH265Dpb * dpb, gint poc_lsb)
{
gint i;
g_return_val_if_fail (dpb != NULL, NULL);
for (i = 0; i < dpb->pic_list->len; i++) {
GstH265Picture *picture =
g_array_index (dpb->pic_list, GstH265Picture *, i);
if (picture->ref && picture->pic_order_cnt_lsb == poc_lsb)
return gst_h265_picture_ref (picture);
}
GST_DEBUG ("No short term reference picture for %d", poc_lsb);
return NULL;
}
/**
* gst_h265_dpb_get_short_ref_by_poc:
* @dpb: a #GstH265Dpb
* @poc: a picture order count
*
* Find a short term reference picture which has matching poc
*
* Returns: (nullable) (transfer full): a #GstH265Picture
*/
GstH265Picture *
gst_h265_dpb_get_short_ref_by_poc (GstH265Dpb * dpb, gint poc)
{
gint i;
g_return_val_if_fail (dpb != NULL, NULL);
for (i = 0; i < dpb->pic_list->len; i++) {
GstH265Picture *picture =
g_array_index (dpb->pic_list, GstH265Picture *, i);
if (picture->ref && !picture->long_term && picture->pic_order_cnt == poc)
return gst_h265_picture_ref (picture);
}
GST_DEBUG ("No short term reference picture for %d", poc);
return NULL;
}
/**
* gst_h265_dpb_get_long_ref_by_poc:
* @dpb: a #GstH265Dpb
* @poc: a picture order count
*
* Find a long term reference picture which has matching poc
*
* Returns: (nullable) (transfer full): a #GstH265Picture
*/
GstH265Picture *
gst_h265_dpb_get_long_ref_by_poc (GstH265Dpb * dpb, gint poc)
{
gint i;
g_return_val_if_fail (dpb != NULL, NULL);
for (i = 0; i < dpb->pic_list->len; i++) {
GstH265Picture *picture =
g_array_index (dpb->pic_list, GstH265Picture *, i);
if (picture->ref && picture->long_term && picture->pic_order_cnt == poc)
return gst_h265_picture_ref (picture);
}
GST_DEBUG ("No long term reference picture for %d", poc);
return NULL;
}
/**
* gst_h265_dpb_get_pictures_all:
* @dpb: a #GstH265Dpb
*
* Return: (element-type GstH265Picture) (transfer full): a #GArray of
* #GstH265Picture stored in @dpb
*/
GArray *
gst_h265_dpb_get_pictures_all (GstH265Dpb * dpb)
{
g_return_val_if_fail (dpb != NULL, NULL);
return g_array_ref (dpb->pic_list);
}
/**
* gst_h265_dpb_get_size:
* @dpb: a #GstH265Dpb
*
* Return: the length of stored dpb array
*/
gint
gst_h265_dpb_get_size (GstH265Dpb * dpb)
{
g_return_val_if_fail (dpb != NULL, -1);
return dpb->pic_list->len;
}
/**
* gst_h265_dpb_get_picture:
* @dpb: a #GstH265Dpb
* @system_frame_number The system frame number
*
* Returns: (transfer full): the picture identified with the specified
* @system_frame_number, or %NULL if DPB does not contain a #GstH265Picture
* corresponding to the @system_frame_number
*
* Since: 1.20
*/
GstH265Picture *
gst_h265_dpb_get_picture (GstH265Dpb * dpb, guint32 system_frame_number)
{
gint i;
g_return_val_if_fail (dpb != NULL, NULL);
for (i = 0; i < dpb->pic_list->len; i++) {
GstH265Picture *picture =
g_array_index (dpb->pic_list, GstH265Picture *, i);
if (picture->system_frame_number == system_frame_number) {
gst_h265_picture_ref (picture);
return picture;
}
}
return NULL;
}
static gboolean
gst_h265_dpb_check_latency_count (GstH265Dpb * dpb, guint32 max_latency)
{
gint i;
for (i = 0; i < dpb->pic_list->len; i++) {
GstH265Picture *picture =
g_array_index (dpb->pic_list, GstH265Picture *, i);
if (!picture->needed_for_output)
continue;
if (picture->pic_latency_cnt >= max_latency)
return TRUE;
}
return FALSE;
}
/**
* gst_h265_dpb_needs_bump:
* @dpb: a #GstH265Dpb
* @max_num_reorder_pics: sps_max_num_reorder_pics[HighestTid]
* @max_latency_increase: SpsMaxLatencyPictures[HighestTid]
* @max_dec_pic_buffering: sps_max_dec_pic_buffering_minus1[HighestTid ] + 1
* or zero if this shouldn't be used for bumping decision
*
* Returns: %TRUE if bumping is required
*
* Since: 1.20
*/
gboolean
gst_h265_dpb_needs_bump (GstH265Dpb * dpb, guint max_num_reorder_pics,
guint max_latency_increase, guint max_dec_pic_buffering)
{
g_return_val_if_fail (dpb != NULL, FALSE);
g_assert (dpb->num_output_needed >= 0);
/* If DPB is full and there is no empty space to store current picture,
* need bumping.
* NOTE: current picture was added already by our decoding flow, so we
* need to do bumping until dpb->pic_list->len == dpb->max_num_pic
*/
if (dpb->pic_list->len > dpb->max_num_pics) {
GST_TRACE ("No empty frame buffer, need bumping");
return TRUE;
}
/* C.5.2.3 */
if (dpb->num_output_needed > max_num_reorder_pics) {
GST_TRACE ("num_output_needed (%d) > max_num_reorder_pics (%d)",
dpb->num_output_needed, max_num_reorder_pics);
return TRUE;
}
if (dpb->num_output_needed && max_latency_increase &&
gst_h265_dpb_check_latency_count (dpb, max_latency_increase)) {
GST_TRACE ("has late picture, max_latency_increase: %d",
max_latency_increase);
return TRUE;
}
/* C.5.2.2 */
if (max_dec_pic_buffering && dpb->pic_list->len >= max_dec_pic_buffering) {
GST_TRACE ("dpb size (%d) >= max_dec_pic_buffering (%d)",
dpb->pic_list->len, max_dec_pic_buffering);
return TRUE;
}
return FALSE;
}
static gint
gst_h265_dpb_get_lowest_output_needed_picture (GstH265Dpb * dpb,
GstH265Picture ** picture)
{
gint i;
GstH265Picture *lowest = NULL;
gint index = -1;
*picture = NULL;
for (i = 0; i < dpb->pic_list->len; i++) {
GstH265Picture *picture =
g_array_index (dpb->pic_list, GstH265Picture *, i);
if (!picture->needed_for_output)
continue;
if (!lowest) {
lowest = picture;
index = i;
continue;
}
if (picture->pic_order_cnt < lowest->pic_order_cnt) {
lowest = picture;
index = i;
}
}
if (lowest)
*picture = gst_h265_picture_ref (lowest);
return index;
}
/**
* gst_h265_dpb_bump:
* @dpb: a #GstH265Dpb
* @drain: whether draining or not
*
* Perform bumping process as defined in C.5.2.4 "Bumping" process.
* If @drain is %TRUE, @dpb will remove a #GstH265Picture from internal array
* so that returned #GstH265Picture could hold the last reference of it
*
* Returns: (nullable) (transfer full): a #GstH265Picture which is needed to be
* outputted
*
* Since: 1.20
*/
GstH265Picture *
gst_h265_dpb_bump (GstH265Dpb * dpb, gboolean drain)
{
GstH265Picture *picture;
gint index;
g_return_val_if_fail (dpb != NULL, NULL);
/* C.5.2.4 "Bumping" process */
index = gst_h265_dpb_get_lowest_output_needed_picture (dpb, &picture);
if (!picture || index < 0)
return NULL;
picture->needed_for_output = FALSE;
dpb->num_output_needed--;
g_assert (dpb->num_output_needed >= 0);
if (!picture->ref || drain)
g_array_remove_index_fast (dpb->pic_list, index);
return picture;
}