mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-11-27 04:01:08 +00:00
Code Issues 1 Projects Releases Wiki Activity
gstreamer/subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth264decoder.c
Nicolas Dufresne 104ae1b2e6 codecs: h264: Save the field_pic_flag 
This flag is set when the stream is interlaced and the specific
slice is made of single parity fields rather the paired at the
macroblock layer. This is rarely needed in late decoding process
but the Rockchip RKVDEC HW interface requires it, hence needs to
be passed through V4L2 Stateless interface.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2009>
2022-03-25 21:40:28 +00:00

3324 lines
105 KiB
C
Raw Blame History

/* GStreamer
* Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
* NOTE: some of implementations are copied/modified from Chromium code
*
* Copyright 2015 The Chromium Authors. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/**
* SECTION:gsth264decoder
* @title: GstH264Decoder
* @short_description: Base class to implement stateless H.264 decoders
* @sources:
* - gsth264picture.h
*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <gst/base/base.h>
#include "gsth264decoder.h"
GST_DEBUG_CATEGORY (gst_h264_decoder_debug);
#define GST_CAT_DEFAULT gst_h264_decoder_debug
typedef enum
{
GST_H264_DECODER_FORMAT_NONE,
GST_H264_DECODER_FORMAT_AVC,
GST_H264_DECODER_FORMAT_BYTE
} GstH264DecoderFormat;
typedef enum
{
GST_H264_DECODER_ALIGN_NONE,
GST_H264_DECODER_ALIGN_NAL,
GST_H264_DECODER_ALIGN_AU
} GstH264DecoderAlign;
struct _GstH264DecoderPrivate
{
GstH264DecoderCompliance compliance;
guint8 profile_idc;
gint width, height;
/* input codec_data, if any */
GstBuffer *codec_data;
guint nal_length_size;
/* state */
GstH264DecoderFormat in_format;
GstH264DecoderAlign align;
GstH264NalParser *parser;
GstH264Dpb *dpb;
/* Cache last field which can not enter the DPB, should be a non ref */
GstH264Picture *last_field;
/* used for low-latency vs. high throughput mode decision */
gboolean is_live;
/* sps/pps of the current slice */
const GstH264SPS *active_sps;
const GstH264PPS *active_pps;
/* Picture currently being processed/decoded */
GstH264Picture *current_picture;
GstVideoCodecFrame *current_frame;
/* Slice (slice header + nalu) currently being processed/decodec */
GstH264Slice current_slice;
gint max_frame_num;
gint max_pic_num;
gint max_long_term_frame_idx;
gint prev_frame_num;
gint prev_ref_frame_num;
gint prev_frame_num_offset;
gboolean prev_has_memmgmnt5;
/* Values related to previously decoded reference picture */
gboolean prev_ref_has_memmgmnt5;
gint prev_ref_top_field_order_cnt;
gint prev_ref_pic_order_cnt_msb;
gint prev_ref_pic_order_cnt_lsb;
GstH264PictureField prev_ref_field;
/* PicOrderCount of the previously outputted frame */
gint last_output_poc;
gboolean process_ref_pic_lists;
guint preferred_output_delay;
/* Reference picture lists, constructed for each frame */
GArray *ref_pic_list_p0;
GArray *ref_pic_list_b0;
GArray *ref_pic_list_b1;
/* Temporary picture list, for reference picture lists in fields,
* corresponding to 8.2.4.2.2 refFrameList0ShortTerm, refFrameList0LongTerm
* and 8.2.4.2.5 refFrameList1ShortTerm and refFrameListLongTerm */
GArray *ref_frame_list_0_short_term;
GArray *ref_frame_list_1_short_term;
GArray *ref_frame_list_long_term;
/* Reference picture lists, constructed for each slice */
GArray *ref_pic_list0;
GArray *ref_pic_list1;
/* For delayed output */
GstQueueArray *output_queue;
};
typedef struct
{
/* Holds ref */
GstVideoCodecFrame *frame;
GstH264Picture *picture;
/* Without ref */
GstH264Decoder *self;
} GstH264DecoderOutputFrame;
#define UPDATE_FLOW_RETURN(ret,new_ret) G_STMT_START { \
if (*(ret) == GST_FLOW_OK) \
*(ret) = new_ret; \
} G_STMT_END
#define parent_class gst_h264_decoder_parent_class
G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstH264Decoder, gst_h264_decoder,
GST_TYPE_VIDEO_DECODER,
G_ADD_PRIVATE (GstH264Decoder);
GST_DEBUG_CATEGORY_INIT (gst_h264_decoder_debug, "h264decoder", 0,
"H.264 Video Decoder"));
static void gst_h264_decoder_finalize (GObject * object);
static gboolean gst_h264_decoder_start (GstVideoDecoder * decoder);
static gboolean gst_h264_decoder_stop (GstVideoDecoder * decoder);
static gboolean gst_h264_decoder_set_format (GstVideoDecoder * decoder,
GstVideoCodecState * state);
static GstFlowReturn gst_h264_decoder_finish (GstVideoDecoder * decoder);
static gboolean gst_h264_decoder_flush (GstVideoDecoder * decoder);
static GstFlowReturn gst_h264_decoder_drain (GstVideoDecoder * decoder);
static GstFlowReturn gst_h264_decoder_handle_frame (GstVideoDecoder * decoder,
GstVideoCodecFrame * frame);
/* codec specific functions */
static GstFlowReturn gst_h264_decoder_process_sps (GstH264Decoder * self,
GstH264SPS * sps);
static GstFlowReturn gst_h264_decoder_decode_slice (GstH264Decoder * self);
static GstFlowReturn gst_h264_decoder_decode_nal (GstH264Decoder * self,
GstH264NalUnit * nalu);
static gboolean gst_h264_decoder_fill_picture_from_slice (GstH264Decoder * self,
const GstH264Slice * slice, GstH264Picture * picture);
static gboolean gst_h264_decoder_calculate_poc (GstH264Decoder * self,
GstH264Picture * picture);
static gboolean gst_h264_decoder_init_gap_picture (GstH264Decoder * self,
GstH264Picture * picture, gint frame_num);
static GstFlowReturn gst_h264_decoder_drain_internal (GstH264Decoder * self);
static void gst_h264_decoder_finish_current_picture (GstH264Decoder * self,
GstFlowReturn * ret);
static void gst_h264_decoder_finish_picture (GstH264Decoder * self,
GstH264Picture * picture, GstFlowReturn * ret);
static void gst_h264_decoder_prepare_ref_pic_lists (GstH264Decoder * self,
GstH264Picture * current_picture);
static void gst_h264_decoder_clear_ref_pic_lists (GstH264Decoder * self);
static gboolean gst_h264_decoder_modify_ref_pic_lists (GstH264Decoder * self);
static gboolean
gst_h264_decoder_sliding_window_picture_marking (GstH264Decoder * self,
GstH264Picture * picture);
static void gst_h264_decoder_do_output_picture (GstH264Decoder * self,
GstH264Picture * picture, GstFlowReturn * ret);
static GstH264Picture *gst_h264_decoder_new_field_picture (GstH264Decoder *
self, GstH264Picture * picture);
static void
gst_h264_decoder_clear_output_frame (GstH264DecoderOutputFrame * output_frame);
enum
{
PROP_0,
PROP_COMPLIANCE,
};
/**
* gst_h264_decoder_compliance_get_type:
*
* Get the compliance type of the h264 decoder.
*
* Since: 1.20
*/
GType
gst_h264_decoder_compliance_get_type (void)
{
static gsize h264_decoder_compliance_type = 0;
static const GEnumValue compliances[] = {
{GST_H264_DECODER_COMPLIANCE_AUTO, "GST_H264_DECODER_COMPLIANCE_AUTO",
"auto"},
{GST_H264_DECODER_COMPLIANCE_STRICT, "GST_H264_DECODER_COMPLIANCE_STRICT",
"strict"},
{GST_H264_DECODER_COMPLIANCE_NORMAL, "GST_H264_DECODER_COMPLIANCE_NORMAL",
"normal"},
{GST_H264_DECODER_COMPLIANCE_FLEXIBLE,
"GST_H264_DECODER_COMPLIANCE_FLEXIBLE", "flexible"},
{0, NULL, NULL},
};
if (g_once_init_enter (&h264_decoder_compliance_type)) {
GType _type;
_type = g_enum_register_static ("GstH264DecoderCompliance", compliances);
g_once_init_leave (&h264_decoder_compliance_type, _type);
}
return (GType) h264_decoder_compliance_type;
}
static void
gst_h264_decoder_get_property (GObject * object, guint property_id,
GValue * value, GParamSpec * pspec)
{
GstH264Decoder *self = GST_H264_DECODER (object);
GstH264DecoderPrivate *priv = self->priv;
switch (property_id) {
case PROP_COMPLIANCE:
GST_OBJECT_LOCK (self);
g_value_set_enum (value, priv->compliance);
GST_OBJECT_UNLOCK (self);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
break;
}
}
static void
gst_h264_decoder_set_property (GObject * object, guint property_id,
const GValue * value, GParamSpec * pspec)
{
GstH264Decoder *self = GST_H264_DECODER (object);
GstH264DecoderPrivate *priv = self->priv;
switch (property_id) {
case PROP_COMPLIANCE:
GST_OBJECT_LOCK (self);
priv->compliance = g_value_get_enum (value);
GST_OBJECT_UNLOCK (self);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
break;
}
}
static void
gst_h264_decoder_class_init (GstH264DecoderClass * klass)
{
GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass);
GObjectClass *object_class = G_OBJECT_CLASS (klass);
object_class->finalize = GST_DEBUG_FUNCPTR (gst_h264_decoder_finalize);
object_class->get_property = gst_h264_decoder_get_property;
object_class->set_property = gst_h264_decoder_set_property;
decoder_class->start = GST_DEBUG_FUNCPTR (gst_h264_decoder_start);
decoder_class->stop = GST_DEBUG_FUNCPTR (gst_h264_decoder_stop);
decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_h264_decoder_set_format);
decoder_class->finish = GST_DEBUG_FUNCPTR (gst_h264_decoder_finish);
decoder_class->flush = GST_DEBUG_FUNCPTR (gst_h264_decoder_flush);
decoder_class->drain = GST_DEBUG_FUNCPTR (gst_h264_decoder_drain);
decoder_class->handle_frame =
GST_DEBUG_FUNCPTR (gst_h264_decoder_handle_frame);
/**
* GstH264Decoder:compliance:
*
* The compliance controls the behavior of the decoder to handle some
* subtle cases and contexts, such as the low-latency DPB bumping or
* mapping the baseline profile as the constrained-baseline profile,
* etc.
*
* Since: 1.20
*/
g_object_class_install_property (object_class, PROP_COMPLIANCE,
g_param_spec_enum ("compliance", "Decoder Compliance",
"The decoder's behavior in compliance with the h264 spec.",
GST_TYPE_H264_DECODER_COMPLIANCE, GST_H264_DECODER_COMPLIANCE_AUTO,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT));
}
static void
gst_h264_decoder_init (GstH264Decoder * self)
{
GstH264DecoderPrivate *priv;
gst_video_decoder_set_packetized (GST_VIDEO_DECODER (self), TRUE);
self->priv = priv = gst_h264_decoder_get_instance_private (self);
priv->last_output_poc = G_MININT32;
priv->ref_pic_list_p0 = g_array_sized_new (FALSE, TRUE,
sizeof (GstH264Picture *), 32);
g_array_set_clear_func (priv->ref_pic_list_p0,
(GDestroyNotify) gst_clear_h264_picture);
priv->ref_pic_list_b0 = g_array_sized_new (FALSE, TRUE,
sizeof (GstH264Picture *), 32);
g_array_set_clear_func (priv->ref_pic_list_b0,
(GDestroyNotify) gst_clear_h264_picture);
priv->ref_pic_list_b1 = g_array_sized_new (FALSE, TRUE,
sizeof (GstH264Picture *), 32);
g_array_set_clear_func (priv->ref_pic_list_b1,
(GDestroyNotify) gst_clear_h264_picture);
priv->ref_frame_list_0_short_term = g_array_sized_new (FALSE, TRUE,
sizeof (GstH264Picture *), 32);
g_array_set_clear_func (priv->ref_frame_list_0_short_term,
(GDestroyNotify) gst_clear_h264_picture);
priv->ref_frame_list_1_short_term = g_array_sized_new (FALSE, TRUE,
sizeof (GstH264Picture *), 32);
g_array_set_clear_func (priv->ref_frame_list_1_short_term,
(GDestroyNotify) gst_clear_h264_picture);
priv->ref_frame_list_long_term = g_array_sized_new (FALSE, TRUE,
sizeof (GstH264Picture *), 32);
g_array_set_clear_func (priv->ref_frame_list_long_term,
(GDestroyNotify) gst_clear_h264_picture);
priv->ref_pic_list0 = g_array_sized_new (FALSE, TRUE,
sizeof (GstH264Picture *), 32);
priv->ref_pic_list1 = g_array_sized_new (FALSE, TRUE,
sizeof (GstH264Picture *), 32);
priv->output_queue =
gst_queue_array_new_for_struct (sizeof (GstH264DecoderOutputFrame), 1);
gst_queue_array_set_clear_func (priv->output_queue,
(GDestroyNotify) gst_h264_decoder_clear_output_frame);
}
static void
gst_h264_decoder_finalize (GObject * object)
{
GstH264Decoder *self = GST_H264_DECODER (object);
GstH264DecoderPrivate *priv = self->priv;
g_array_unref (priv->ref_pic_list_p0);
g_array_unref (priv->ref_pic_list_b0);
g_array_unref (priv->ref_pic_list_b1);
g_array_unref (priv->ref_frame_list_0_short_term);
g_array_unref (priv->ref_frame_list_1_short_term);
g_array_unref (priv->ref_frame_list_long_term);
g_array_unref (priv->ref_pic_list0);
g_array_unref (priv->ref_pic_list1);
gst_queue_array_free (priv->output_queue);
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static void
gst_h264_decoder_reset (GstH264Decoder * self)
{
GstH264DecoderPrivate *priv = self->priv;
gst_clear_buffer (&priv->codec_data);
g_clear_pointer (&self->input_state, gst_video_codec_state_unref);
g_clear_pointer (&priv->parser, gst_h264_nal_parser_free);
g_clear_pointer (&priv->dpb, gst_h264_dpb_free);
gst_clear_h264_picture (&priv->last_field);
priv->profile_idc = 0;
priv->width = 0;
priv->height = 0;
priv->nal_length_size = 4;
}
static gboolean
gst_h264_decoder_start (GstVideoDecoder * decoder)
{
GstH264Decoder *self = GST_H264_DECODER (decoder);
GstH264DecoderPrivate *priv = self->priv;
gst_h264_decoder_reset (self);
priv->parser = gst_h264_nal_parser_new ();
priv->dpb = gst_h264_dpb_new ();
return TRUE;
}
static gboolean
gst_h264_decoder_stop (GstVideoDecoder * decoder)
{
GstH264Decoder *self = GST_H264_DECODER (decoder);
gst_h264_decoder_reset (self);
return TRUE;
}
static void
gst_h264_decoder_clear_output_frame (GstH264DecoderOutputFrame * output_frame)
{
if (!output_frame)
return;
if (output_frame->frame) {
gst_video_decoder_release_frame (GST_VIDEO_DECODER (output_frame->self),
output_frame->frame);
output_frame->frame = NULL;
}
gst_clear_h264_picture (&output_frame->picture);
}
static void
gst_h264_decoder_clear_dpb (GstH264Decoder * self, gboolean flush)
{
GstVideoDecoder *decoder = GST_VIDEO_DECODER (self);
GstH264DecoderPrivate *priv = self->priv;
GstH264Picture *picture;
/* If we are not flushing now, videodecoder baseclass will hold
* GstVideoCodecFrame. Release frames manually */
if (!flush) {
while ((picture = gst_h264_dpb_bump (priv->dpb, TRUE)) != NULL) {
GstVideoCodecFrame *frame = gst_video_decoder_get_frame (decoder,
picture->system_frame_number);
if (frame)
gst_video_decoder_release_frame (decoder, frame);
gst_h264_picture_unref (picture);
}
}
gst_queue_array_clear (priv->output_queue);
gst_h264_decoder_clear_ref_pic_lists (self);
gst_clear_h264_picture (&priv->last_field);
gst_h264_dpb_clear (priv->dpb);
priv->last_output_poc = G_MININT32;
}
static gboolean
gst_h264_decoder_flush (GstVideoDecoder * decoder)
{
GstH264Decoder *self = GST_H264_DECODER (decoder);
gst_h264_decoder_clear_dpb (self, TRUE);
return TRUE;
}
static GstFlowReturn
gst_h264_decoder_drain (GstVideoDecoder * decoder)
{
GstH264Decoder *self = GST_H264_DECODER (decoder);
/* dpb will be cleared by this method */
return gst_h264_decoder_drain_internal (self);
}
static GstFlowReturn
gst_h264_decoder_finish (GstVideoDecoder * decoder)
{
return gst_h264_decoder_drain (decoder);
}
static GstFlowReturn
gst_h264_decoder_handle_frame (GstVideoDecoder * decoder,
GstVideoCodecFrame * frame)
{
GstH264Decoder *self = GST_H264_DECODER (decoder);
GstH264DecoderPrivate *priv = self->priv;
GstBuffer *in_buf = frame->input_buffer;
GstH264NalUnit nalu;
GstH264ParserResult pres;
GstMapInfo map;
GstFlowReturn decode_ret = GST_FLOW_OK;
GST_LOG_OBJECT (self,
"handle frame, PTS: %" GST_TIME_FORMAT ", DTS: %"
GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_PTS (in_buf)),
GST_TIME_ARGS (GST_BUFFER_DTS (in_buf)));
priv->current_frame = frame;
gst_buffer_map (in_buf, &map, GST_MAP_READ);
if (priv->in_format == GST_H264_DECODER_FORMAT_AVC) {
pres = gst_h264_parser_identify_nalu_avc (priv->parser,
map.data, 0, map.size, priv->nal_length_size, &nalu);
while (pres == GST_H264_PARSER_OK && decode_ret == GST_FLOW_OK) {
decode_ret = gst_h264_decoder_decode_nal (self, &nalu);
pres = gst_h264_parser_identify_nalu_avc (priv->parser,
map.data, nalu.offset + nalu.size, map.size, priv->nal_length_size,
&nalu);
}
} else {
pres = gst_h264_parser_identify_nalu (priv->parser,
map.data, 0, map.size, &nalu);
if (pres == GST_H264_PARSER_NO_NAL_END)
pres = GST_H264_PARSER_OK;
while (pres == GST_H264_PARSER_OK && decode_ret == GST_FLOW_OK) {
decode_ret = gst_h264_decoder_decode_nal (self, &nalu);
pres = gst_h264_parser_identify_nalu (priv->parser,
map.data, nalu.offset + nalu.size, map.size, &nalu);
if (pres == GST_H264_PARSER_NO_NAL_END)
pres = GST_H264_PARSER_OK;
}
}
gst_buffer_unmap (in_buf, &map);
if (decode_ret != GST_FLOW_OK) {
if (decode_ret == GST_FLOW_ERROR) {
GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE,
("Failed to decode data"), (NULL), decode_ret);
}
gst_video_decoder_drop_frame (decoder, frame);
gst_clear_h264_picture (&priv->current_picture);
priv->current_frame = NULL;
return decode_ret;
}
gst_h264_decoder_finish_current_picture (self, &decode_ret);
gst_video_codec_frame_unref (frame);
priv->current_frame = NULL;
if (decode_ret == GST_FLOW_ERROR) {
GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE,
("Failed to decode data"), (NULL), decode_ret);
}
return decode_ret;
}
static GstFlowReturn
gst_h264_decoder_parse_sps (GstH264Decoder * self, GstH264NalUnit * nalu)
{
GstH264DecoderPrivate *priv = self->priv;
GstH264SPS sps;
GstH264ParserResult pres;
GstFlowReturn ret;
pres = gst_h264_parse_sps (nalu, &sps);
if (pres != GST_H264_PARSER_OK) {
GST_WARNING_OBJECT (self, "Failed to parse SPS, result %d", pres);
return GST_FLOW_ERROR;
}
GST_LOG_OBJECT (self, "SPS parsed");
ret = gst_h264_decoder_process_sps (self, &sps);
if (ret != GST_FLOW_OK) {
GST_WARNING_OBJECT (self, "Failed to process SPS");
} else if (gst_h264_parser_update_sps (priv->parser,
&sps) != GST_H264_PARSER_OK) {
GST_WARNING_OBJECT (self, "Failed to update SPS");
ret = GST_FLOW_ERROR;
}
gst_h264_sps_clear (&sps);
return ret;
}
static GstFlowReturn
gst_h264_decoder_parse_pps (GstH264Decoder * self, GstH264NalUnit * nalu)
{
GstH264DecoderPrivate *priv = self->priv;
GstH264PPS pps;
GstH264ParserResult pres;
GstFlowReturn ret = GST_FLOW_OK;
pres = gst_h264_parse_pps (priv->parser, nalu, &pps);
if (pres != GST_H264_PARSER_OK) {
GST_WARNING_OBJECT (self, "Failed to parse PPS, result %d", pres);
return GST_FLOW_ERROR;
}
GST_LOG_OBJECT (self, "PPS parsed");
if (pps.num_slice_groups_minus1 > 0) {
GST_FIXME_OBJECT (self, "FMO is not supported");
ret = GST_FLOW_ERROR;
} else if (gst_h264_parser_update_pps (priv->parser, &pps)
!= GST_H264_PARSER_OK) {
GST_WARNING_OBJECT (self, "Failed to update PPS");
ret = GST_FLOW_ERROR;
}
gst_h264_pps_clear (&pps);
return ret;
}
static GstFlowReturn
gst_h264_decoder_parse_codec_data (GstH264Decoder * self, const guint8 * data,
gsize size)
{
GstH264DecoderPrivate *priv = self->priv;
guint num_sps, num_pps;
guint off;
gint i;
GstH264ParserResult pres;
GstH264NalUnit nalu;
GstFlowReturn ret = GST_FLOW_OK;
#ifndef GST_DISABLE_GST_DEBUG
guint profile;
#endif
/* parse the avcC data */
if (size < 7) { /* when numSPS==0 and numPPS==0, length is 7 bytes */
return GST_FLOW_ERROR;
}
/* parse the version, this must be 1 */
if (data[0] != 1) {
return GST_FLOW_ERROR;
}
#ifndef GST_DISABLE_GST_DEBUG
/* AVCProfileIndication */
/* profile_compat */
/* AVCLevelIndication */
profile = (data[1] << 16) | (data[2] << 8) | data[3];
GST_DEBUG_OBJECT (self, "profile %06x", profile);
#endif
/* 6 bits reserved | 2 bits lengthSizeMinusOne */
/* this is the number of bytes in front of the NAL units to mark their
* length */
priv->nal_length_size = (data[4] & 0x03) + 1;
GST_DEBUG_OBJECT (self, "nal length size %u", priv->nal_length_size);
num_sps = data[5] & 0x1f;
off = 6;
for (i = 0; i < num_sps; i++) {
pres = gst_h264_parser_identify_nalu_avc (priv->parser,
data, off, size, 2, &nalu);
if (pres != GST_H264_PARSER_OK) {
GST_WARNING_OBJECT (self, "Failed to identify SPS nalu");
return GST_FLOW_ERROR;
}
ret = gst_h264_decoder_parse_sps (self, &nalu);
if (ret != GST_FLOW_OK) {
GST_WARNING_OBJECT (self, "Failed to parse SPS");
return ret;
}
off = nalu.offset + nalu.size;
}
if (off >= size) {
GST_WARNING_OBJECT (self, "Too small avcC");
return GST_FLOW_ERROR;
}
num_pps = data[off];
off++;
for (i = 0; i < num_pps; i++) {
pres = gst_h264_parser_identify_nalu_avc (priv->parser,
data, off, size, 2, &nalu);
if (pres != GST_H264_PARSER_OK) {
GST_WARNING_OBJECT (self, "Failed to identify PPS nalu");
return GST_FLOW_ERROR;
}
ret = gst_h264_decoder_parse_pps (self, &nalu);
if (ret != GST_FLOW_OK) {
GST_WARNING_OBJECT (self, "Failed to parse PPS");
return ret;
}
off = nalu.offset + nalu.size;
}
return GST_FLOW_OK;
}
static gboolean
gst_h264_decoder_preprocess_slice (GstH264Decoder * self, GstH264Slice * slice)
{
GstH264DecoderPrivate *priv = self->priv;
if (!priv->current_picture) {
if (slice->header.first_mb_in_slice != 0) {
GST_ERROR_OBJECT (self, "Invalid stream, first_mb_in_slice %d",
slice->header.first_mb_in_slice);
return FALSE;
}
}
return TRUE;
}
static void
gst_h264_decoder_update_pic_nums (GstH264Decoder * self,
GstH264Picture * current_picture, gint frame_num)
{
GstH264DecoderPrivate *priv = self->priv;
GArray *dpb = gst_h264_dpb_get_pictures_all (priv->dpb);
gint i;
for (i = 0; i < dpb->len; i++) {
GstH264Picture *picture = g_array_index (dpb, GstH264Picture *, i);
if (!GST_H264_PICTURE_IS_REF (picture))
continue;
if (GST_H264_PICTURE_IS_LONG_TERM_REF (picture)) {
if (GST_H264_PICTURE_IS_FRAME (current_picture))
picture->long_term_pic_num = picture->long_term_frame_idx;
else if (current_picture->field == picture->field)
picture->long_term_pic_num = 2 * picture->long_term_frame_idx + 1;
else
picture->long_term_pic_num = 2 * picture->long_term_frame_idx;
} else {
if (picture->frame_num > frame_num)
picture->frame_num_wrap = picture->frame_num - priv->max_frame_num;
else
picture->frame_num_wrap = picture->frame_num;
if (GST_H264_PICTURE_IS_FRAME (current_picture))
picture->pic_num = picture->frame_num_wrap;
else if (picture->field == current_picture->field)
picture->pic_num = 2 * picture->frame_num_wrap + 1;
else
picture->pic_num = 2 * picture->frame_num_wrap;
}
}
g_array_unref (dpb);
}
static GstH264Picture *
gst_h264_decoder_split_frame (GstH264Decoder * self, GstH264Picture * picture)
{
GstH264Picture *other_field;
g_assert (GST_H264_PICTURE_IS_FRAME (picture));
other_field = gst_h264_decoder_new_field_picture (self, picture);
if (!other_field) {
GST_WARNING_OBJECT (self,
"Couldn't split frame into complementary field pair");
return NULL;
}
GST_LOG_OBJECT (self, "Split picture %p, poc %d, frame num %d",
picture, picture->pic_order_cnt, picture->frame_num);
/* FIXME: enhance TFF decision by using picture timing SEI */
if (picture->top_field_order_cnt < picture->bottom_field_order_cnt) {
picture->field = GST_H264_PICTURE_FIELD_TOP_FIELD;
picture->pic_order_cnt = picture->top_field_order_cnt;
other_field->field = GST_H264_PICTURE_FIELD_BOTTOM_FIELD;
other_field->pic_order_cnt = picture->bottom_field_order_cnt;
} else {
picture->field = GST_H264_PICTURE_FIELD_BOTTOM_FIELD;
picture->pic_order_cnt = picture->bottom_field_order_cnt;
other_field->field = GST_H264_PICTURE_FIELD_TOP_FIELD;
other_field->pic_order_cnt = picture->top_field_order_cnt;
}
other_field->top_field_order_cnt = picture->top_field_order_cnt;
other_field->bottom_field_order_cnt = picture->bottom_field_order_cnt;
other_field->frame_num = picture->frame_num;
other_field->ref = picture->ref;
other_field->nonexisting = picture->nonexisting;
other_field->system_frame_number = picture->system_frame_number;
other_field->field_pic_flag = picture->field_pic_flag;
return other_field;
}
static void
output_picture_directly (GstH264Decoder * self, GstH264Picture * picture,
GstFlowReturn * ret)
{
GstH264DecoderPrivate *priv = self->priv;
GstH264Picture *out_pic = NULL;
GstFlowReturn flow_ret = GST_FLOW_OK;
g_assert (ret != NULL);
if (GST_H264_PICTURE_IS_FRAME (picture)) {
g_assert (priv->last_field == NULL);
out_pic = g_steal_pointer (&picture);
goto output;
}
if (priv->last_field == NULL) {
if (picture->second_field) {
GST_WARNING ("Set the last output %p poc:%d, without first field",
picture, picture->pic_order_cnt);
flow_ret = GST_FLOW_ERROR;
goto output;
}
/* Just cache the first field. */
priv->last_field = g_steal_pointer (&picture);
} else {
if (!picture->second_field || !picture->other_field
|| picture->other_field != priv->last_field) {
GST_WARNING ("The last field %p poc:%d is not the pair of the "
"current field %p poc:%d",
priv->last_field, priv->last_field->pic_order_cnt,
picture, picture->pic_order_cnt);
gst_clear_h264_picture (&priv->last_field);
flow_ret = GST_FLOW_ERROR;
goto output;
}
GST_TRACE ("Pair the last field %p poc:%d and the current"
" field %p poc:%d",
priv->last_field, priv->last_field->pic_order_cnt,
picture, picture->pic_order_cnt);
out_pic = priv->last_field;
priv->last_field = NULL;
/* Link each field. */
out_pic->other_field = picture;
}
output:
if (out_pic) {
gst_h264_dpb_set_last_output (priv->dpb, out_pic);
gst_h264_decoder_do_output_picture (self, out_pic, &flow_ret);
}
gst_clear_h264_picture (&picture);
UPDATE_FLOW_RETURN (ret, flow_ret);
}
static void
add_picture_to_dpb (GstH264Decoder * self, GstH264Picture * picture)
{
GstH264DecoderPrivate *priv = self->priv;
if (!gst_h264_dpb_get_interlaced (priv->dpb)) {
g_assert (priv->last_field == NULL);
gst_h264_dpb_add (priv->dpb, picture);
return;
}
/* The first field of the last picture may not be able to enter the
DPB if it is a non ref, but if the second field enters the DPB, we
need to add both of them. */
if (priv->last_field && picture->other_field == priv->last_field) {
gst_h264_dpb_add (priv->dpb, priv->last_field);
priv->last_field = NULL;
}
gst_h264_dpb_add (priv->dpb, picture);
}
static void
_bump_dpb (GstH264Decoder * self, GstH264DpbBumpMode bump_level,
GstH264Picture * current_picture, GstFlowReturn * ret)
{
GstH264DecoderPrivate *priv = self->priv;
g_assert (ret != NULL);
while (gst_h264_dpb_needs_bump (priv->dpb, current_picture, bump_level)) {
GstH264Picture *to_output;
to_output = gst_h264_dpb_bump (priv->dpb, FALSE);
if (!to_output) {
GST_WARNING_OBJECT (self, "Bumping is needed but no picture to output");
break;
}
gst_h264_decoder_do_output_picture (self, to_output, ret);
}
}
static GstFlowReturn
gst_h264_decoder_handle_frame_num_gap (GstH264Decoder * self, gint frame_num)
{
GstH264DecoderPrivate *priv = self->priv;
const GstH264SPS *sps = priv->active_sps;
gint unused_short_term_frame_num;
if (!sps) {
GST_ERROR_OBJECT (self, "No active sps");
return GST_FLOW_ERROR;
}
if (priv->prev_ref_frame_num == frame_num) {
GST_TRACE_OBJECT (self,
"frame_num == PrevRefFrameNum (%d), not a gap", frame_num);
return GST_FLOW_OK;
}
if (((priv->prev_ref_frame_num + 1) % priv->max_frame_num) == frame_num) {
GST_TRACE_OBJECT (self,
"frame_num == (PrevRefFrameNum + 1) %% MaxFrameNum (%d), not a gap",
frame_num);
return GST_FLOW_OK;
}
if (gst_h264_dpb_get_size (priv->dpb) == 0) {
GST_TRACE_OBJECT (self, "DPB is empty, not a gap");
return GST_FLOW_OK;
}
if (!sps->gaps_in_frame_num_value_allowed_flag) {
/* This is likely the case where some frames were dropped.
* then we need to keep decoding without error out */
GST_WARNING_OBJECT (self, "Invalid frame num %d, maybe frame drop",
frame_num);
return GST_FLOW_OK;
}
GST_DEBUG_OBJECT (self, "Handling frame num gap %d -> %d (MaxFrameNum: %d)",
priv->prev_ref_frame_num, frame_num, priv->max_frame_num);
/* 7.4.3/7-23 */
unused_short_term_frame_num =
(priv->prev_ref_frame_num + 1) % priv->max_frame_num;
while (unused_short_term_frame_num != frame_num) {
GstH264Picture *picture = gst_h264_picture_new ();
GstFlowReturn ret = GST_FLOW_OK;
if (!gst_h264_decoder_init_gap_picture (self, picture,
unused_short_term_frame_num))
return GST_FLOW_ERROR;
gst_h264_decoder_update_pic_nums (self, picture,
unused_short_term_frame_num);
/* C.2.1 */
if (!gst_h264_decoder_sliding_window_picture_marking (self, picture)) {
GST_ERROR_OBJECT (self,
"Couldn't perform sliding window picture marking");
return GST_FLOW_ERROR;
}
gst_h264_dpb_delete_unused (priv->dpb);
_bump_dpb (self, GST_H264_DPB_BUMP_NORMAL_LATENCY, picture, &ret);
if (ret != GST_FLOW_OK)
return ret;
/* the picture is short term ref, add to DPB. */
if (gst_h264_dpb_get_interlaced (priv->dpb)) {
GstH264Picture *other_field =
gst_h264_decoder_split_frame (self, picture);
add_picture_to_dpb (self, picture);
add_picture_to_dpb (self, other_field);
} else {
add_picture_to_dpb (self, picture);
}
unused_short_term_frame_num++;
unused_short_term_frame_num %= priv->max_frame_num;
}
return GST_FLOW_OK;
}
static gboolean
gst_h264_decoder_init_current_picture (GstH264Decoder * self)
{
GstH264DecoderPrivate *priv = self->priv;
if (!gst_h264_decoder_fill_picture_from_slice (self, &priv->current_slice,
priv->current_picture)) {
return FALSE;
}
if (!gst_h264_decoder_calculate_poc (self, priv->current_picture))
return FALSE;
/* If the slice header indicates we will have to perform reference marking
* process after this picture is decoded, store required data for that
* purpose */
if (priv->current_slice.header.
dec_ref_pic_marking.adaptive_ref_pic_marking_mode_flag) {
priv->current_picture->dec_ref_pic_marking =
priv->current_slice.header.dec_ref_pic_marking;
}
return TRUE;
}
static GstFlowReturn
gst_h264_decoder_start_current_picture (GstH264Decoder * self)
{
GstH264DecoderClass *klass;
GstH264DecoderPrivate *priv = self->priv;
const GstH264SPS *sps;
gint frame_num;
GstFlowReturn ret = GST_FLOW_OK;
GstH264Picture *current_picture;
g_assert (priv->current_picture != NULL);
g_assert (priv->active_sps != NULL);
g_assert (priv->active_pps != NULL);
sps = priv->active_sps;
priv->max_frame_num = sps->max_frame_num;
frame_num = priv->current_slice.header.frame_num;
if (priv->current_slice.nalu.idr_pic_flag)
priv->prev_ref_frame_num = 0;
ret = gst_h264_decoder_handle_frame_num_gap (self, frame_num);
if (ret != GST_FLOW_OK)
return ret;
if (!gst_h264_decoder_init_current_picture (self))
return GST_FLOW_ERROR;
current_picture = priv->current_picture;
/* If the new picture is an IDR, flush DPB */
if (current_picture->idr) {
if (!current_picture->dec_ref_pic_marking.no_output_of_prior_pics_flag) {
ret = gst_h264_decoder_drain_internal (self);
if (ret != GST_FLOW_OK)
return ret;
} else {
/* C.4.4 Removal of pictures from the DPB before possible insertion
* of the current picture
*
* If decoded picture is IDR and no_output_of_prior_pics_flag is equal to 1
* or is inferred to be equal to 1, all frame buffers in the DPB
* are emptied without output of the pictures they contain,
* and DPB fullness is set to 0.
*/
gst_h264_decoder_clear_dpb (self, FALSE);
}
}
gst_h264_decoder_update_pic_nums (self, current_picture, frame_num);
if (priv->process_ref_pic_lists)
gst_h264_decoder_prepare_ref_pic_lists (self, current_picture);
klass = GST_H264_DECODER_GET_CLASS (self);
if (klass->start_picture) {
ret = klass->start_picture (self, priv->current_picture,
&priv->current_slice, priv->dpb);
if (ret != GST_FLOW_OK) {
GST_WARNING_OBJECT (self, "subclass does not want to start picture");
return ret;
}
}
return GST_FLOW_OK;
}
static GstH264Picture *
gst_h264_decoder_new_field_picture (GstH264Decoder * self,
GstH264Picture * picture)
{
GstH264DecoderClass *klass = GST_H264_DECODER_GET_CLASS (self);
GstH264Picture *new_picture;
if (!klass->new_field_picture) {
GST_WARNING_OBJECT (self, "Subclass does not support interlaced stream");
return NULL;
}
new_picture = gst_h264_picture_new ();
/* don't confuse subclass by non-existing picture */
if (!picture->nonexisting) {
GstFlowReturn ret;
ret = klass->new_field_picture (self, picture, new_picture);
if (ret != GST_FLOW_OK) {
GST_WARNING_OBJECT (self, "Subclass couldn't handle new field picture");
gst_h264_picture_unref (new_picture);
return NULL;
}
}
new_picture->other_field = picture;
new_picture->second_field = TRUE;
return new_picture;
}
static gboolean
gst_h264_decoder_find_first_field_picture (GstH264Decoder * self,
GstH264Slice * slice, GstH264Picture ** first_field)
{
GstH264DecoderPrivate *priv = self->priv;
const GstH264SliceHdr *slice_hdr = &slice->header;
GstH264Picture *prev_field;
gboolean in_dpb;
*first_field = NULL;
prev_field = NULL;
in_dpb = FALSE;
if (gst_h264_dpb_get_interlaced (priv->dpb)) {
if (priv->last_field) {
prev_field = priv->last_field;
in_dpb = FALSE;
} else if (gst_h264_dpb_get_size (priv->dpb) > 0) {
GstH264Picture *prev_picture;
GArray *pictures;
pictures = gst_h264_dpb_get_pictures_all (priv->dpb);
prev_picture =
g_array_index (pictures, GstH264Picture *, pictures->len - 1);
g_array_unref (pictures); /* prev_picture should be held */
/* Previous picture was a field picture. */
if (!GST_H264_PICTURE_IS_FRAME (prev_picture)
&& !prev_picture->other_field) {
prev_field = prev_picture;
in_dpb = TRUE;
}
}
} else {
g_assert (priv->last_field == NULL);
}
/* This is not a field picture */
if (!slice_hdr->field_pic_flag) {
if (!prev_field)
return TRUE;
GST_WARNING_OBJECT (self, "Previous picture %p (poc %d) is not complete",
prev_field, prev_field->pic_order_cnt);
goto error;
}
/* OK, this is the first field. */
if (!prev_field)
return TRUE;
if (prev_field->frame_num != slice_hdr->frame_num) {
GST_WARNING_OBJECT (self, "Previous picture %p (poc %d) is not complete",
prev_field, prev_field->pic_order_cnt);
goto error;
} else {
GstH264PictureField current_field = slice_hdr->bottom_field_flag ?
GST_H264_PICTURE_FIELD_BOTTOM_FIELD : GST_H264_PICTURE_FIELD_TOP_FIELD;
if (current_field == prev_field->field) {
GST_WARNING_OBJECT (self,
"Currnet picture and previous picture have identical field %d",
current_field);
goto error;
}
}
*first_field = gst_h264_picture_ref (prev_field);
return TRUE;
error:
if (!in_dpb) {
gst_clear_h264_picture (&priv->last_field);
} else {
/* FIXME: implement fill gap field picture if it is already in DPB */
}
return FALSE;
}
static GstFlowReturn
gst_h264_decoder_parse_slice (GstH264Decoder * self, GstH264NalUnit * nalu)
{
GstH264DecoderPrivate *priv = self->priv;
GstH264ParserResult pres = GST_H264_PARSER_OK;
GstFlowReturn ret = GST_FLOW_OK;
memset (&priv->current_slice, 0, sizeof (GstH264Slice));
pres = gst_h264_parser_parse_slice_hdr (priv->parser, nalu,
&priv->current_slice.header, TRUE, TRUE);
if (pres != GST_H264_PARSER_OK) {
GST_ERROR_OBJECT (self, "Failed to parse slice header, ret %d", pres);
memset (&priv->current_slice, 0, sizeof (GstH264Slice));
return GST_FLOW_ERROR;
}
priv->current_slice.nalu = *nalu;
if (!gst_h264_decoder_preprocess_slice (self, &priv->current_slice))
return GST_FLOW_ERROR;
priv->active_pps = priv->current_slice.header.pps;
priv->active_sps = priv->active_pps->sequence;
/* Check whether field picture boundary within given codec frame.
* This might happen in case that upstream sent buffer per frame unit,
* not picture unit (i.e., AU unit).
* If AU boundary is detected, then finish first field picture we decoded
* in this chain, we should finish the current picture and
* start new field picture decoding */
if (gst_h264_dpb_get_interlaced (priv->dpb) && priv->current_picture &&
!GST_H264_PICTURE_IS_FRAME (priv->current_picture) &&
!priv->current_picture->second_field) {
GstH264PictureField prev_field = priv->current_picture->field;
GstH264PictureField cur_field = GST_H264_PICTURE_FIELD_FRAME;
if (priv->current_slice.header.field_pic_flag)
cur_field = priv->current_slice.header.bottom_field_flag ?
GST_H264_PICTURE_FIELD_BOTTOM_FIELD :
GST_H264_PICTURE_FIELD_TOP_FIELD;
if (cur_field != prev_field) {
GST_LOG_OBJECT (self,
"Found new field picture, finishing the first field picture");
gst_h264_decoder_finish_current_picture (self, &ret);
}
}
if (!priv->current_picture) {
GstH264DecoderClass *klass = GST_H264_DECODER_GET_CLASS (self);
GstH264Picture *picture = NULL;
GstH264Picture *first_field = NULL;
GstFlowReturn ret = GST_FLOW_OK;
g_assert (priv->current_frame);
if (!gst_h264_decoder_find_first_field_picture (self,
&priv->current_slice, &first_field)) {
GST_ERROR_OBJECT (self, "Couldn't find or determine first picture");
return GST_FLOW_ERROR;
}
if (first_field) {
picture = gst_h264_decoder_new_field_picture (self, first_field);
gst_h264_picture_unref (first_field);
if (!picture) {
GST_ERROR_OBJECT (self, "Couldn't duplicate the first field picture");
return GST_FLOW_ERROR;
}
} else {
picture = gst_h264_picture_new ();
if (klass->new_picture)
ret = klass->new_picture (self, priv->current_frame, picture);
if (ret != GST_FLOW_OK) {
GST_WARNING_OBJECT (self, "subclass does not want accept new picture");
priv->current_picture = NULL;
gst_h264_picture_unref (picture);
return ret;
}
}
/* This allows accessing the frame from the picture. */
picture->system_frame_number = priv->current_frame->system_frame_number;
priv->current_picture = picture;
ret = gst_h264_decoder_start_current_picture (self);
if (ret != GST_FLOW_OK) {
GST_WARNING_OBJECT (self, "start picture failed");
return ret;
}
}
return gst_h264_decoder_decode_slice (self);
}
static GstFlowReturn
gst_h264_decoder_decode_nal (GstH264Decoder * self, GstH264NalUnit * nalu)
{
GstFlowReturn ret = GST_FLOW_OK;
GST_LOG_OBJECT (self, "Parsed nal type: %d, offset %d, size %d",
nalu->type, nalu->offset, nalu->size);
switch (nalu->type) {
case GST_H264_NAL_SPS:
ret = gst_h264_decoder_parse_sps (self, nalu);
break;
case GST_H264_NAL_PPS:
ret = gst_h264_decoder_parse_pps (self, nalu);
break;
case GST_H264_NAL_SLICE:
case GST_H264_NAL_SLICE_DPA:
case GST_H264_NAL_SLICE_DPB:
case GST_H264_NAL_SLICE_DPC:
case GST_H264_NAL_SLICE_IDR:
case GST_H264_NAL_SLICE_EXT:
ret = gst_h264_decoder_parse_slice (self, nalu);
break;
default:
break;
}
return ret;
}
static void
gst_h264_decoder_format_from_caps (GstH264Decoder * self, GstCaps * caps,
GstH264DecoderFormat * format, GstH264DecoderAlign * align)
{
if (format)
*format = GST_H264_DECODER_FORMAT_NONE;
if (align)
*align = GST_H264_DECODER_ALIGN_NONE;
if (!gst_caps_is_fixed (caps)) {
GST_WARNING_OBJECT (self, "Caps wasn't fixed");
return;
}
GST_DEBUG_OBJECT (self, "parsing caps: %" GST_PTR_FORMAT, caps);
if (caps && gst_caps_get_size (caps) > 0) {
GstStructure *s = gst_caps_get_structure (caps, 0);
const gchar *str = NULL;
if (format) {
if ((str = gst_structure_get_string (s, "stream-format"))) {
if (strcmp (str, "avc") == 0 || strcmp (str, "avc3") == 0)
*format = GST_H264_DECODER_FORMAT_AVC;
else if (strcmp (str, "byte-stream") == 0)
*format = GST_H264_DECODER_FORMAT_BYTE;
}
}
if (align) {
if ((str = gst_structure_get_string (s, "alignment"))) {
if (strcmp (str, "au") == 0)
*align = GST_H264_DECODER_ALIGN_AU;
else if (strcmp (str, "nal") == 0)
*align = GST_H264_DECODER_ALIGN_NAL;
}
}
}
}
static gboolean
gst_h264_decoder_set_format (GstVideoDecoder * decoder,
GstVideoCodecState * state)
{
GstH264Decoder *self = GST_H264_DECODER (decoder);
GstH264DecoderPrivate *priv = self->priv;
GstQuery *query;
GST_DEBUG_OBJECT (decoder, "Set format");
if (self->input_state)
gst_video_codec_state_unref (self->input_state);
self->input_state = gst_video_codec_state_ref (state);
if (state->caps) {
GstStructure *str;
const GValue *codec_data_value;
GstH264DecoderFormat format;
GstH264DecoderAlign align;
gst_h264_decoder_format_from_caps (self, state->caps, &format, &align);
str = gst_caps_get_structure (state->caps, 0);
codec_data_value = gst_structure_get_value (str, "codec_data");
if (GST_VALUE_HOLDS_BUFFER (codec_data_value)) {
gst_buffer_replace (&priv->codec_data,
gst_value_get_buffer (codec_data_value));
} else {
gst_buffer_replace (&priv->codec_data, NULL);
}
if (format == GST_H264_DECODER_FORMAT_NONE) {
/* codec_data implies avc */
if (codec_data_value != NULL) {
GST_WARNING_OBJECT (self,
"video/x-h264 caps with codec_data but no stream-format=avc");
format = GST_H264_DECODER_FORMAT_AVC;
} else {
/* otherwise assume bytestream input */
GST_WARNING_OBJECT (self,
"video/x-h264 caps without codec_data or stream-format");
format = GST_H264_DECODER_FORMAT_BYTE;
}
}
if (format == GST_H264_DECODER_FORMAT_AVC) {
/* AVC requires codec_data, AVC3 might have one and/or SPS/PPS inline */
if (codec_data_value == NULL) {
/* Try it with size 4 anyway */
priv->nal_length_size = 4;
GST_WARNING_OBJECT (self,
"avc format without codec data, assuming nal length size is 4");
}
/* AVC implies alignment=au */
if (align == GST_H264_DECODER_ALIGN_NONE)
align = GST_H264_DECODER_ALIGN_AU;
}
if (format == GST_H264_DECODER_FORMAT_BYTE) {
if (codec_data_value != NULL) {
GST_WARNING_OBJECT (self, "bytestream with codec data");
}
}
priv->in_format = format;
priv->align = align;
}
if (priv->codec_data) {
GstMapInfo map;
gst_buffer_map (priv->codec_data, &map, GST_MAP_READ);
if (gst_h264_decoder_parse_codec_data (self, map.data, map.size) !=
GST_FLOW_OK) {
/* keep going without error.
* Probably inband SPS/PPS might be valid data */
GST_WARNING_OBJECT (self, "Failed to handle codec data");
}
gst_buffer_unmap (priv->codec_data, &map);
}
/* in case live streaming, we will run on low-latency mode */
priv->is_live = FALSE;
query = gst_query_new_latency ();
if (gst_pad_peer_query (GST_VIDEO_DECODER_SINK_PAD (self), query))
gst_query_parse_latency (query, &priv->is_live, NULL, NULL);
gst_query_unref (query);
if (priv->is_live)
GST_DEBUG_OBJECT (self, "Live source, will run on low-latency mode");
return TRUE;
}
static gboolean
gst_h264_decoder_fill_picture_from_slice (GstH264Decoder * self,
const GstH264Slice * slice, GstH264Picture * picture)
{
GstH264DecoderClass *klass = GST_H264_DECODER_GET_CLASS (self);
const GstH264SliceHdr *slice_hdr = &slice->header;
const GstH264PPS *pps;
const GstH264SPS *sps;
pps = slice_hdr->pps;
if (!pps) {
GST_ERROR_OBJECT (self, "No pps in slice header");
return FALSE;
}
sps = pps->sequence;
if (!sps) {
GST_ERROR_OBJECT (self, "No sps in pps");
return FALSE;
}
picture->idr = slice->nalu.idr_pic_flag;
picture->dec_ref_pic_marking = slice_hdr->dec_ref_pic_marking;
picture->field_pic_flag = slice_hdr->field_pic_flag;
if (picture->idr)
picture->idr_pic_id = slice_hdr->idr_pic_id;
if (slice_hdr->field_pic_flag)
picture->field =
slice_hdr->bottom_field_flag ?
GST_H264_PICTURE_FIELD_BOTTOM_FIELD : GST_H264_PICTURE_FIELD_TOP_FIELD;
else
picture->field = GST_H264_PICTURE_FIELD_FRAME;
if (!GST_H264_PICTURE_IS_FRAME (picture) && !klass->new_field_picture) {
GST_FIXME_OBJECT (self, "Subclass doesn't support interlace stream");
return FALSE;
}
picture->nal_ref_idc = slice->nalu.ref_idc;
if (slice->nalu.ref_idc != 0)
gst_h264_picture_set_reference (picture,
GST_H264_PICTURE_REF_SHORT_TERM, FALSE);
picture->frame_num = slice_hdr->frame_num;
/* 7.4.3 */
if (!slice_hdr->field_pic_flag)
picture->pic_num = slice_hdr->frame_num;
else
picture->pic_num = 2 * slice_hdr->frame_num + 1;
picture->pic_order_cnt_type = sps->pic_order_cnt_type;
switch (picture->pic_order_cnt_type) {
case 0:
picture->pic_order_cnt_lsb = slice_hdr->pic_order_cnt_lsb;
picture->delta_pic_order_cnt_bottom =
slice_hdr->delta_pic_order_cnt_bottom;
break;
case 1:
picture->delta_pic_order_cnt0 = slice_hdr->delta_pic_order_cnt[0];
picture->delta_pic_order_cnt1 = slice_hdr->delta_pic_order_cnt[1];
break;
case 2:
break;
default:
g_assert_not_reached ();
return FALSE;
}
return TRUE;
}
static gboolean
gst_h264_decoder_calculate_poc (GstH264Decoder * self, GstH264Picture * picture)
{
GstH264DecoderPrivate *priv = self->priv;
const GstH264SPS *sps = priv->active_sps;
if (!sps) {
GST_ERROR_OBJECT (self, "No active SPS");
return FALSE;
}
switch (picture->pic_order_cnt_type) {
case 0:{
/* See spec 8.2.1.1 */
gint prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb;
gint max_pic_order_cnt_lsb;
if (picture->idr) {
prev_pic_order_cnt_msb = prev_pic_order_cnt_lsb = 0;
} else {
if (priv->prev_ref_has_memmgmnt5) {
if (priv->prev_ref_field != GST_H264_PICTURE_FIELD_BOTTOM_FIELD) {
prev_pic_order_cnt_msb = 0;
prev_pic_order_cnt_lsb = priv->prev_ref_top_field_order_cnt;
} else {
prev_pic_order_cnt_msb = 0;
prev_pic_order_cnt_lsb = 0;
}
} else {
prev_pic_order_cnt_msb = priv->prev_ref_pic_order_cnt_msb;
prev_pic_order_cnt_lsb = priv->prev_ref_pic_order_cnt_lsb;
}
}
max_pic_order_cnt_lsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
if ((picture->pic_order_cnt_lsb < prev_pic_order_cnt_lsb) &&
(prev_pic_order_cnt_lsb - picture->pic_order_cnt_lsb >=
max_pic_order_cnt_lsb / 2)) {
picture->pic_order_cnt_msb =
prev_pic_order_cnt_msb + max_pic_order_cnt_lsb;
} else if ((picture->pic_order_cnt_lsb > prev_pic_order_cnt_lsb)
&& (picture->pic_order_cnt_lsb - prev_pic_order_cnt_lsb >
max_pic_order_cnt_lsb / 2)) {
picture->pic_order_cnt_msb =
prev_pic_order_cnt_msb - max_pic_order_cnt_lsb;
} else {
picture->pic_order_cnt_msb = prev_pic_order_cnt_msb;
}
if (picture->field != GST_H264_PICTURE_FIELD_BOTTOM_FIELD) {
picture->top_field_order_cnt =
picture->pic_order_cnt_msb + picture->pic_order_cnt_lsb;
}
switch (picture->field) {
case GST_H264_PICTURE_FIELD_FRAME:
picture->top_field_order_cnt = picture->pic_order_cnt_msb +
picture->pic_order_cnt_lsb;
picture->bottom_field_order_cnt = picture->top_field_order_cnt +
picture->delta_pic_order_cnt_bottom;
break;
case GST_H264_PICTURE_FIELD_TOP_FIELD:
picture->top_field_order_cnt = picture->pic_order_cnt_msb +
picture->pic_order_cnt_lsb;
break;
case GST_H264_PICTURE_FIELD_BOTTOM_FIELD:
picture->bottom_field_order_cnt = picture->pic_order_cnt_msb +
picture->pic_order_cnt_lsb;
break;
}
break;
}
case 1:{
gint abs_frame_num = 0;
gint expected_pic_order_cnt = 0;
gint i;
/* See spec 8.2.1.2 */
if (priv->prev_has_memmgmnt5)
priv->prev_frame_num_offset = 0;
if (picture->idr)
picture->frame_num_offset = 0;
else if (priv->prev_frame_num > picture->frame_num)
picture->frame_num_offset =
priv->prev_frame_num_offset + priv->max_frame_num;
else
picture->frame_num_offset = priv->prev_frame_num_offset;
if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0)
abs_frame_num = picture->frame_num_offset + picture->frame_num;
else
abs_frame_num = 0;
if (picture->nal_ref_idc == 0 && abs_frame_num > 0)
--abs_frame_num;
if (abs_frame_num > 0) {
gint pic_order_cnt_cycle_cnt, frame_num_in_pic_order_cnt_cycle;
gint expected_delta_per_pic_order_cnt_cycle = 0;
if (sps->num_ref_frames_in_pic_order_cnt_cycle == 0) {
GST_WARNING_OBJECT (self,
"Invalid num_ref_frames_in_pic_order_cnt_cycle in stream");
return FALSE;
}
pic_order_cnt_cycle_cnt =
(abs_frame_num - 1) / sps->num_ref_frames_in_pic_order_cnt_cycle;
frame_num_in_pic_order_cnt_cycle =
(abs_frame_num - 1) % sps->num_ref_frames_in_pic_order_cnt_cycle;
for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++) {
expected_delta_per_pic_order_cnt_cycle +=
sps->offset_for_ref_frame[i];
}
expected_pic_order_cnt = pic_order_cnt_cycle_cnt *
expected_delta_per_pic_order_cnt_cycle;
/* frame_num_in_pic_order_cnt_cycle is verified < 255 in parser */
for (i = 0; i <= frame_num_in_pic_order_cnt_cycle; ++i)
expected_pic_order_cnt += sps->offset_for_ref_frame[i];
}
if (!picture->nal_ref_idc)
expected_pic_order_cnt += sps->offset_for_non_ref_pic;
if (GST_H264_PICTURE_IS_FRAME (picture)) {
picture->top_field_order_cnt =
expected_pic_order_cnt + picture->delta_pic_order_cnt0;
picture->bottom_field_order_cnt = picture->top_field_order_cnt +
sps->offset_for_top_to_bottom_field + picture->delta_pic_order_cnt1;
} else if (picture->field != GST_H264_PICTURE_FIELD_BOTTOM_FIELD) {
picture->top_field_order_cnt =
expected_pic_order_cnt + picture->delta_pic_order_cnt0;
} else {
picture->bottom_field_order_cnt = expected_pic_order_cnt +
sps->offset_for_top_to_bottom_field + picture->delta_pic_order_cnt0;
}
break;
}
case 2:{
gint temp_pic_order_cnt;
/* See spec 8.2.1.3 */
if (priv->prev_has_memmgmnt5)
priv->prev_frame_num_offset = 0;
if (picture->idr)
picture->frame_num_offset = 0;
else if (priv->prev_frame_num > picture->frame_num)
picture->frame_num_offset =
priv->prev_frame_num_offset + priv->max_frame_num;
else
picture->frame_num_offset = priv->prev_frame_num_offset;
if (picture->idr) {
temp_pic_order_cnt = 0;
} else if (!picture->nal_ref_idc) {
temp_pic_order_cnt =
2 * (picture->frame_num_offset + picture->frame_num) - 1;
} else {
temp_pic_order_cnt =
2 * (picture->frame_num_offset + picture->frame_num);
}
if (GST_H264_PICTURE_IS_FRAME (picture)) {
picture->top_field_order_cnt = temp_pic_order_cnt;
picture->bottom_field_order_cnt = temp_pic_order_cnt;
} else if (picture->field == GST_H264_PICTURE_FIELD_BOTTOM_FIELD) {
picture->bottom_field_order_cnt = temp_pic_order_cnt;
} else {
picture->top_field_order_cnt = temp_pic_order_cnt;
}
break;
}
default:
GST_WARNING_OBJECT (self,
"Invalid pic_order_cnt_type: %d", sps->pic_order_cnt_type);
return FALSE;
}
switch (picture->field) {
case GST_H264_PICTURE_FIELD_FRAME:
picture->pic_order_cnt =
MIN (picture->top_field_order_cnt, picture->bottom_field_order_cnt);
break;
case GST_H264_PICTURE_FIELD_TOP_FIELD:
picture->pic_order_cnt = picture->top_field_order_cnt;
break;
case GST_H264_PICTURE_FIELD_BOTTOM_FIELD:
picture->pic_order_cnt = picture->bottom_field_order_cnt;
break;
default:
g_assert_not_reached ();
return FALSE;
}
return TRUE;
}
static void
gst_h264_decoder_drain_output_queue (GstH264Decoder * self, guint num,
GstFlowReturn * ret)
{
GstH264DecoderPrivate *priv = self->priv;
GstH264DecoderClass *klass = GST_H264_DECODER_GET_CLASS (self);
g_assert (klass->output_picture);
g_assert (ret != NULL);
while (gst_queue_array_get_length (priv->output_queue) > num) {
GstH264DecoderOutputFrame *output_frame = (GstH264DecoderOutputFrame *)
gst_queue_array_pop_head_struct (priv->output_queue);
GstFlowReturn flow_ret = klass->output_picture (self, output_frame->frame,
output_frame->picture);
UPDATE_FLOW_RETURN (ret, flow_ret);
}
}
static void
gst_h264_decoder_do_output_picture (GstH264Decoder * self,
GstH264Picture * picture, GstFlowReturn * ret)
{
GstH264DecoderPrivate *priv = self->priv;
GstVideoCodecFrame *frame = NULL;
GstH264DecoderOutputFrame output_frame;
GstFlowReturn flow_ret = GST_FLOW_OK;
g_assert (ret != NULL);
GST_LOG_OBJECT (self, "Outputting picture %p (frame_num %d, poc %d)",
picture, picture->frame_num, picture->pic_order_cnt);
if (picture->pic_order_cnt < priv->last_output_poc) {
GST_WARNING_OBJECT (self,
"Outputting out of order %d -> %d, likely a broken stream",
priv->last_output_poc, picture->pic_order_cnt);
}
priv->last_output_poc = picture->pic_order_cnt;
frame = gst_video_decoder_get_frame (GST_VIDEO_DECODER (self),
picture->system_frame_number);
if (!frame) {
GST_ERROR_OBJECT (self,
"No available codec frame with frame number %d",
picture->system_frame_number);
UPDATE_FLOW_RETURN (ret, GST_FLOW_ERROR);
gst_h264_picture_unref (picture);
return;
}
output_frame.frame = frame;
output_frame.picture = picture;
output_frame.self = self;
gst_queue_array_push_tail_struct (priv->output_queue, &output_frame);
gst_h264_decoder_drain_output_queue (self, priv->preferred_output_delay,
&flow_ret);
UPDATE_FLOW_RETURN (ret, flow_ret);
}
static void
gst_h264_decoder_finish_current_picture (GstH264Decoder * self,
GstFlowReturn * ret)
{
GstH264DecoderPrivate *priv = self->priv;
GstH264DecoderClass *klass;
GstFlowReturn flow_ret = GST_FLOW_OK;
if (!priv->current_picture)
return;
klass = GST_H264_DECODER_GET_CLASS (self);
if (klass->end_picture) {
flow_ret = klass->end_picture (self, priv->current_picture);
if (flow_ret != GST_FLOW_OK) {
GST_WARNING_OBJECT (self,
"end picture failed, marking picture %p non-existing "
"(frame_num %d, poc %d)", priv->current_picture,
priv->current_picture->frame_num,
priv->current_picture->pic_order_cnt);
priv->current_picture->nonexisting = TRUE;
/* this fake nonexisting picture will not trigger ouput_picture() */
gst_video_decoder_drop_frame (GST_VIDEO_DECODER (self),
gst_video_codec_frame_ref (priv->current_frame));
}
}
/* We no longer need the per frame reference lists */
gst_h264_decoder_clear_ref_pic_lists (self);
/* finish picture takes ownership of the picture */
gst_h264_decoder_finish_picture (self, priv->current_picture, &flow_ret);
priv->current_picture = NULL;
UPDATE_FLOW_RETURN (ret, flow_ret);
}
static gint
poc_asc_compare (const GstH264Picture ** a, const GstH264Picture ** b)
{
return (*a)->pic_order_cnt - (*b)->pic_order_cnt;
}
static gint
poc_desc_compare (const GstH264Picture ** a, const GstH264Picture ** b)
{
return (*b)->pic_order_cnt - (*a)->pic_order_cnt;
}
static GstFlowReturn
gst_h264_decoder_drain_internal (GstH264Decoder * self)
{
GstH264DecoderPrivate *priv = self->priv;
GstH264Picture *picture;
GstFlowReturn ret = GST_FLOW_OK;
while ((picture = gst_h264_dpb_bump (priv->dpb, TRUE)) != NULL) {
gst_h264_decoder_do_output_picture (self, picture, &ret);
}
gst_h264_decoder_drain_output_queue (self, 0, &ret);
gst_clear_h264_picture (&priv->last_field);
gst_h264_dpb_clear (priv->dpb);
priv->last_output_poc = G_MININT32;
return ret;
}
static gboolean
gst_h264_decoder_handle_memory_management_opt (GstH264Decoder * self,
GstH264Picture * picture)
{
GstH264DecoderPrivate *priv = self->priv;
gint i;
for (i = 0; i < G_N_ELEMENTS (picture->dec_ref_pic_marking.ref_pic_marking);
i++) {
GstH264RefPicMarking *ref_pic_marking =
&picture->dec_ref_pic_marking.ref_pic_marking[i];
guint8 type = ref_pic_marking->memory_management_control_operation;
GST_TRACE_OBJECT (self, "memory management operation %d, type %d", i, type);
/* Normal end of operations' specification */
if (type == 0)
return TRUE;
switch (type) {
case 4:
priv->max_long_term_frame_idx =
ref_pic_marking->max_long_term_frame_idx_plus1 - 1;
break;
case 5:
priv->max_long_term_frame_idx = -1;
break;
default:
break;
}
if (!gst_h264_dpb_perform_memory_management_control_operation (priv->dpb,
ref_pic_marking, picture)) {
GST_WARNING_OBJECT (self, "memory management operation type %d failed",
type);
/* Most likely our implementation fault, but let's just perform
* next MMCO if any */
}
}
return TRUE;
}
static gboolean
gst_h264_decoder_sliding_window_picture_marking (GstH264Decoder * self,
GstH264Picture * picture)
{
GstH264DecoderPrivate *priv = self->priv;
const GstH264SPS *sps = priv->active_sps;
gint num_ref_pics;
gint max_num_ref_frames;
/* Skip this for the second field */
if (picture->second_field)
return TRUE;
if (!sps) {
GST_ERROR_OBJECT (self, "No active sps");
return FALSE;
}
/* 8.2.5.3. Ensure the DPB doesn't overflow by discarding the oldest picture */
num_ref_pics = gst_h264_dpb_num_ref_frames (priv->dpb);
max_num_ref_frames = MAX (1, sps->num_ref_frames);
if (num_ref_pics < max_num_ref_frames)
return TRUE;
/* In theory, num_ref_pics shouldn't be larger than max_num_ref_frames
* but it could happen if our implementation is wrong somehow or so.
* Just try to remove reference pictures as many as possible in order to
* avoid DPB overflow.
*/
while (num_ref_pics >= max_num_ref_frames) {
/* Max number of reference pics reached, need to remove one of the short
* term ones. Find smallest frame_num_wrap short reference picture and mark
* it as unused */
GstH264Picture *to_unmark =
gst_h264_dpb_get_lowest_frame_num_short_ref (priv->dpb);
if (num_ref_pics > max_num_ref_frames) {
GST_WARNING_OBJECT (self,
"num_ref_pics %d is larger than allowed maximum %d",
num_ref_pics, max_num_ref_frames);
}
if (!to_unmark) {
GST_WARNING_OBJECT (self, "Could not find a short ref picture to unmark");
return FALSE;
}
GST_TRACE_OBJECT (self,
"Unmark reference flag of picture %p (frame_num %d, poc %d)",
to_unmark, to_unmark->frame_num, to_unmark->pic_order_cnt);
gst_h264_picture_set_reference (to_unmark, GST_H264_PICTURE_REF_NONE, TRUE);
gst_h264_picture_unref (to_unmark);
num_ref_pics--;
}
return TRUE;
}
/* This method ensures that DPB does not overflow, either by removing
* reference pictures as specified in the stream, or using a sliding window
* procedure to remove the oldest one.
* It also performs marking and unmarking pictures as reference.
* See spac 8.2.5.1 */
static gboolean
gst_h264_decoder_reference_picture_marking (GstH264Decoder * self,
GstH264Picture * picture)
{
GstH264DecoderPrivate *priv = self->priv;
/* If the current picture is an IDR, all reference pictures are unmarked */
if (picture->idr) {
gst_h264_dpb_mark_all_non_ref (priv->dpb);
if (picture->dec_ref_pic_marking.long_term_reference_flag) {
gst_h264_picture_set_reference (picture,
GST_H264_PICTURE_REF_LONG_TERM, FALSE);
picture->long_term_frame_idx = 0;
priv->max_long_term_frame_idx = 0;
} else {
gst_h264_picture_set_reference (picture,
GST_H264_PICTURE_REF_SHORT_TERM, FALSE);
priv->max_long_term_frame_idx = -1;
}
return TRUE;
}
/* Not an IDR. If the stream contains instructions on how to discard pictures
* from DPB and how to mark/unmark existing reference pictures, do so.
* Otherwise, fall back to default sliding window process */
if (picture->dec_ref_pic_marking.adaptive_ref_pic_marking_mode_flag) {
if (picture->nonexisting) {
GST_WARNING_OBJECT (self,
"Invalid memory management operation for non-existing picture "
"%p (frame_num %d, poc %d", picture, picture->frame_num,
picture->pic_order_cnt);
}
return gst_h264_decoder_handle_memory_management_opt (self, picture);
}
return gst_h264_decoder_sliding_window_picture_marking (self, picture);
}
static GstH264DpbBumpMode
get_bump_level (GstH264Decoder * self)
{
GstH264DecoderPrivate *priv = self->priv;
/* User set the mode explicitly. */
switch (priv->compliance) {
case GST_H264_DECODER_COMPLIANCE_STRICT:
return GST_H264_DPB_BUMP_NORMAL_LATENCY;
case GST_H264_DECODER_COMPLIANCE_NORMAL:
return GST_H264_DPB_BUMP_LOW_LATENCY;
case GST_H264_DECODER_COMPLIANCE_FLEXIBLE:
return GST_H264_DPB_BUMP_VERY_LOW_LATENCY;
default:
break;
}
/* GST_H264_DECODER_COMPLIANCE_AUTO case. */
if (priv->is_live) {
/* The baseline and constrained-baseline profiles do not have B frames
and do not use the picture reorder, safe to use the higher bump level. */
if (priv->profile_idc == GST_H264_PROFILE_BASELINE)
return GST_H264_DPB_BUMP_VERY_LOW_LATENCY;
return GST_H264_DPB_BUMP_LOW_LATENCY;
}
return GST_H264_DPB_BUMP_NORMAL_LATENCY;
}
static void
gst_h264_decoder_finish_picture (GstH264Decoder * self,
GstH264Picture * picture, GstFlowReturn * ret)
{
GstVideoDecoder *decoder = GST_VIDEO_DECODER (self);
GstH264DecoderPrivate *priv = self->priv;
GstH264DpbBumpMode bump_level = get_bump_level (self);
/* Finish processing the picture.
* Start by storing previous picture data for later use */
if (picture->ref) {
gst_h264_decoder_reference_picture_marking (self, picture);
priv->prev_ref_has_memmgmnt5 = picture->mem_mgmt_5;
priv->prev_ref_top_field_order_cnt = picture->top_field_order_cnt;
priv->prev_ref_pic_order_cnt_msb = picture->pic_order_cnt_msb;
priv->prev_ref_pic_order_cnt_lsb = picture->pic_order_cnt_lsb;
priv->prev_ref_field = picture->field;
priv->prev_ref_frame_num = picture->frame_num;
}
priv->prev_frame_num = picture->frame_num;
priv->prev_has_memmgmnt5 = picture->mem_mgmt_5;
priv->prev_frame_num_offset = picture->frame_num_offset;
/* Remove unused (for reference or later output) pictures from DPB, marking
* them as such */
gst_h264_dpb_delete_unused (priv->dpb);
/* If field pictures belong to different codec frame,
* drop codec frame of the second field because we are consuming
* only the first codec frame via GstH264Decoder::output_picture() method */
if (picture->second_field && picture->other_field &&
picture->system_frame_number !=
picture->other_field->system_frame_number) {
GstVideoCodecFrame *frame = gst_video_decoder_get_frame (decoder,
picture->system_frame_number);
gst_video_decoder_release_frame (decoder, frame);
}
/* C.4.4 */
if (picture->mem_mgmt_5) {
GstFlowReturn drain_ret;
GST_TRACE_OBJECT (self, "Memory management type 5, drain the DPB");
drain_ret = gst_h264_decoder_drain_internal (self);
UPDATE_FLOW_RETURN (ret, drain_ret);
}
_bump_dpb (self, bump_level, picture, ret);
/* Add a ref to avoid the case of directly outputed and destroyed. */
gst_h264_picture_ref (picture);
/* C.4.5.1, C.4.5.2
- If the current decoded picture is the second field of a complementary
reference field pair, add to DPB.
C.4.5.1
For A reference decoded picture, the "bumping" process is invoked
repeatedly until there is an empty frame buffer, then add to DPB:
C.4.5.2
For a non-reference decoded picture, if there is empty frame buffer
after bumping the smaller POC, add to DPB.
Otherwise, output directly. */
if ((picture->second_field && picture->other_field
&& picture->other_field->ref)
|| picture->ref || gst_h264_dpb_has_empty_frame_buffer (priv->dpb)) {
/* Split frame into top/bottom field pictures for reference picture marking
* process. Even if current picture has field_pic_flag equal to zero,
* if next picture is a field picture, complementary field pair of reference
* frame should have individual pic_num and long_term_pic_num.
*/
if (gst_h264_dpb_get_interlaced (priv->dpb) &&
GST_H264_PICTURE_IS_FRAME (picture)) {
GstH264Picture *other_field =
gst_h264_decoder_split_frame (self, picture);
add_picture_to_dpb (self, picture);
if (!other_field) {
GST_WARNING_OBJECT (self,
"Couldn't split frame into complementary field pair");
/* Keep decoding anyway... */
} else {
add_picture_to_dpb (self, other_field);
}
} else {
add_picture_to_dpb (self, picture);
}
} else {
output_picture_directly (self, picture, ret);
}
GST_LOG_OBJECT (self,
"Finishing picture %p (frame_num %d, poc %d), entries in DPB %d",
picture, picture->frame_num, picture->pic_order_cnt,
gst_h264_dpb_get_size (priv->dpb));
gst_h264_picture_unref (picture);
/* For the live mode, we try to bump here to avoid waiting
for another decoding circle. */
if (priv->is_live && priv->compliance != GST_H264_DECODER_COMPLIANCE_STRICT)
_bump_dpb (self, bump_level, NULL, ret);
}
static gboolean
gst_h264_decoder_update_max_num_reorder_frames (GstH264Decoder * self,
GstH264SPS * sps)
{
GstH264DecoderPrivate *priv = self->priv;
gsize max_num_reorder_frames = 0;
if (sps->vui_parameters_present_flag
&& sps->vui_parameters.bitstream_restriction_flag) {
max_num_reorder_frames = sps->vui_parameters.num_reorder_frames;
if (max_num_reorder_frames > gst_h264_dpb_get_max_num_frames (priv->dpb)) {
GST_WARNING
("max_num_reorder_frames present, but larger than MaxDpbFrames (%d > %d)",
(gint) max_num_reorder_frames,
gst_h264_dpb_get_max_num_frames (priv->dpb));
max_num_reorder_frames = 0;
return FALSE;
}
gst_h264_dpb_set_max_num_reorder_frames (priv->dpb, max_num_reorder_frames);
return TRUE;
}
if (priv->compliance == GST_H264_DECODER_COMPLIANCE_STRICT) {
gst_h264_dpb_set_max_num_reorder_frames (priv->dpb,
gst_h264_dpb_get_max_num_frames (priv->dpb));
return TRUE;
}
/* max_num_reorder_frames not present, infer it from profile/constraints. */
if (sps->profile_idc == 66 || sps->profile_idc == 83) {
/* baseline, constrained baseline and scalable-baseline profiles
only contain I/P frames. */
max_num_reorder_frames = 0;
} else if (sps->constraint_set3_flag) {
/* constraint_set3_flag may mean the -intra only profile. */
switch (sps->profile_idc) {
case 44:
case 86:
case 100:
case 110:
case 122:
case 244:
max_num_reorder_frames = 0;
break;
default:
max_num_reorder_frames = gst_h264_dpb_get_max_num_frames (priv->dpb);
break;
}
} else {
max_num_reorder_frames = gst_h264_dpb_get_max_num_frames (priv->dpb);
}
gst_h264_dpb_set_max_num_reorder_frames (priv->dpb, max_num_reorder_frames);
return TRUE;
}
typedef enum
{
GST_H264_LEVEL_L1 = 10,
GST_H264_LEVEL_L1B = 9,
GST_H264_LEVEL_L1_1 = 11,
GST_H264_LEVEL_L1_2 = 12,
GST_H264_LEVEL_L1_3 = 13,
GST_H264_LEVEL_L2_0 = 20,
GST_H264_LEVEL_L2_1 = 21,
GST_H264_LEVEL_L2_2 = 22,
GST_H264_LEVEL_L3 = 30,
GST_H264_LEVEL_L3_1 = 31,
GST_H264_LEVEL_L3_2 = 32,
GST_H264_LEVEL_L4 = 40,
GST_H264_LEVEL_L4_1 = 41,
GST_H264_LEVEL_L4_2 = 42,
GST_H264_LEVEL_L5 = 50,
GST_H264_LEVEL_L5_1 = 51,
GST_H264_LEVEL_L5_2 = 52,
GST_H264_LEVEL_L6 = 60,
GST_H264_LEVEL_L6_1 = 61,
GST_H264_LEVEL_L6_2 = 62,
} GstH264DecoderLevel;
typedef struct
{
GstH264DecoderLevel level;
guint32 max_mbps;
guint32 max_fs;
guint32 max_dpb_mbs;
guint32 max_main_br;
} LevelLimits;
static const LevelLimits level_limits_map[] = {
{GST_H264_LEVEL_L1, 1485, 99, 396, 64},
{GST_H264_LEVEL_L1B, 1485, 99, 396, 128},
{GST_H264_LEVEL_L1_1, 3000, 396, 900, 192},
{GST_H264_LEVEL_L1_2, 6000, 396, 2376, 384},
{GST_H264_LEVEL_L1_3, 11800, 396, 2376, 768},
{GST_H264_LEVEL_L2_0, 11880, 396, 2376, 2000},
{GST_H264_LEVEL_L2_1, 19800, 792, 4752, 4000},
{GST_H264_LEVEL_L2_2, 20250, 1620, 8100, 4000},
{GST_H264_LEVEL_L3, 40500, 1620, 8100, 10000},
{GST_H264_LEVEL_L3_1, 108000, 3600, 18000, 14000},
{GST_H264_LEVEL_L3_2, 216000, 5120, 20480, 20000},
{GST_H264_LEVEL_L4, 245760, 8192, 32768, 20000},
{GST_H264_LEVEL_L4_1, 245760, 8192, 32768, 50000},
{GST_H264_LEVEL_L4_2, 522240, 8704, 34816, 50000},
{GST_H264_LEVEL_L5, 589824, 22080, 110400, 135000},
{GST_H264_LEVEL_L5_1, 983040, 36864, 184320, 240000},
{GST_H264_LEVEL_L5_2, 2073600, 36864, 184320, 240000},
{GST_H264_LEVEL_L6, 4177920, 139264, 696320, 240000},
{GST_H264_LEVEL_L6_1, 8355840, 139264, 696320, 480000},
{GST_H264_LEVEL_L6_2, 16711680, 139264, 696320, 800000}
};
static gint
h264_level_to_max_dpb_mbs (GstH264DecoderLevel level)
{
gint i;
for (i = 0; i < G_N_ELEMENTS (level_limits_map); i++) {
if (level == level_limits_map[i].level)
return level_limits_map[i].max_dpb_mbs;
}
return 0;
}
static void
gst_h264_decoder_set_latency (GstH264Decoder * self, const GstH264SPS * sps,
gint max_dpb_size)
{
GstH264DecoderPrivate *priv = self->priv;
GstCaps *caps;
GstClockTime min, max;
GstStructure *structure;
gint fps_d = 1, fps_n = 0;
GstH264DpbBumpMode bump_level;
guint32 frames_delay;
caps = gst_pad_get_current_caps (GST_VIDEO_DECODER_SRC_PAD (self));
if (!caps)
return;
structure = gst_caps_get_structure (caps, 0);
if (gst_structure_get_fraction (structure, "framerate", &fps_n, &fps_d)) {
if (fps_n == 0) {
/* variable framerate: see if we have a max-framerate */
gst_structure_get_fraction (structure, "max-framerate", &fps_n, &fps_d);
}
}
gst_caps_unref (caps);
/* if no fps or variable, then 25/1 */
if (fps_n == 0) {
fps_n = 25;
fps_d = 1;
}
bump_level = get_bump_level (self);
frames_delay = 0;
switch (bump_level) {
case GST_H264_DPB_BUMP_NORMAL_LATENCY:
/* We always wait the DPB full before bumping. */
frames_delay = max_dpb_size;
break;
case GST_H264_DPB_BUMP_LOW_LATENCY:
/* We bump the IDR if the second frame is not a minus POC. */
frames_delay = 1;
break;
case GST_H264_DPB_BUMP_VERY_LOW_LATENCY:
/* We bump the IDR immediately. */
frames_delay = 0;
break;
default:
g_assert_not_reached ();
break;
}
/* Consider output delay wanted by subclass */
frames_delay += priv->preferred_output_delay;
min = gst_util_uint64_scale_int (frames_delay * GST_SECOND, fps_d, fps_n);
max = gst_util_uint64_scale_int ((max_dpb_size + priv->preferred_output_delay)
* GST_SECOND, fps_d, fps_n);
GST_LOG_OBJECT (self,
"latency min %" G_GUINT64_FORMAT " max %" G_GUINT64_FORMAT, min, max);
gst_video_decoder_set_latency (GST_VIDEO_DECODER (self), min, max);
}
static GstFlowReturn
gst_h264_decoder_process_sps (GstH264Decoder * self, GstH264SPS * sps)
{
GstH264DecoderClass *klass = GST_H264_DECODER_GET_CLASS (self);
GstH264DecoderPrivate *priv = self->priv;
guint8 level;
gint max_dpb_mbs;
gint width_mb, height_mb;
gint max_dpb_frames;
gint max_dpb_size;
gint prev_max_dpb_size;
gboolean prev_interlaced;
gboolean interlaced;
GstFlowReturn ret = GST_FLOW_OK;
if (sps->frame_mbs_only_flag == 0) {
if (!klass->new_field_picture) {
GST_FIXME_OBJECT (self,
"frame_mbs_only_flag != 1 not supported by subclass");
return GST_FLOW_NOT_NEGOTIATED;
}
if (sps->mb_adaptive_frame_field_flag) {
GST_LOG_OBJECT (self,
"mb_adaptive_frame_field_flag == 1, MBAFF sequence");
} else {
GST_LOG_OBJECT (self, "mb_adaptive_frame_field_flag == 0, PAFF sequence");
}
}
interlaced = !sps->frame_mbs_only_flag;
/* Spec A.3.1 and A.3.2
* For Baseline, Constrained Baseline and Main profile, the indicated level is
* Level 1b if level_idc is equal to 11 and constraint_set3_flag is equal to 1
*/
level = sps->level_idc;
if (level == 11 && (sps->profile_idc == 66 || sps->profile_idc == 77) &&
sps->constraint_set3_flag) {
/* Level 1b */
level = 9;
}
max_dpb_mbs = h264_level_to_max_dpb_mbs ((GstH264DecoderLevel) level);
if (!max_dpb_mbs)
return GST_FLOW_ERROR;
width_mb = sps->width / 16;
height_mb = sps->height / 16;
max_dpb_frames = MIN (max_dpb_mbs / (width_mb * height_mb),
GST_H264_DPB_MAX_SIZE);
if (sps->vui_parameters_present_flag
&& sps->vui_parameters.bitstream_restriction_flag)
max_dpb_frames = MAX (1, sps->vui_parameters.max_dec_frame_buffering);
/* Case 1) There might be some non-conforming streams that require more DPB
* size than that of specified one by SPS
* Case 2) If bitstream_restriction_flag is not present,
* max_dec_frame_buffering should be inferred
* to be equal to MaxDpbFrames, then MaxDpbFrames can exceed num_ref_frames
* See https://chromium-review.googlesource.com/c/chromium/src/+/760276/
*/
max_dpb_size = MAX (max_dpb_frames, sps->num_ref_frames);
if (max_dpb_size > GST_H264_DPB_MAX_SIZE) {
GST_WARNING_OBJECT (self, "Too large calculated DPB size %d", max_dpb_size);
max_dpb_size = GST_H264_DPB_MAX_SIZE;
}
/* Safety, so that subclass don't need bound checking */
g_return_val_if_fail (max_dpb_size <= GST_H264_DPB_MAX_SIZE, GST_FLOW_ERROR);
prev_max_dpb_size = gst_h264_dpb_get_max_num_frames (priv->dpb);
prev_interlaced = gst_h264_dpb_get_interlaced (priv->dpb);
if (priv->width != sps->width || priv->height != sps->height ||
prev_max_dpb_size != max_dpb_size || prev_interlaced != interlaced) {
GstH264DecoderClass *klass = GST_H264_DECODER_GET_CLASS (self);
GST_DEBUG_OBJECT (self,
"SPS updated, resolution: %dx%d -> %dx%d, dpb size: %d -> %d, "
"interlaced %d -> %d",
priv->width, priv->height, sps->width, sps->height,
prev_max_dpb_size, max_dpb_size, prev_interlaced, interlaced);
ret = gst_h264_decoder_drain (GST_VIDEO_DECODER (self));
if (ret != GST_FLOW_OK)
return ret;
g_assert (klass->new_sequence);
if (klass->get_preferred_output_delay) {
priv->preferred_output_delay =
klass->get_preferred_output_delay (self, priv->is_live);
} else {
priv->preferred_output_delay = 0;
}
ret = klass->new_sequence (self,
sps, max_dpb_size + priv->preferred_output_delay);
if (ret != GST_FLOW_OK) {
GST_WARNING_OBJECT (self, "subclass does not want accept new sequence");
return ret;
}
priv->profile_idc = sps->profile_idc;
priv->width = sps->width;
priv->height = sps->height;
gst_h264_decoder_set_latency (self, sps, max_dpb_size);
gst_h264_dpb_set_max_num_frames (priv->dpb, max_dpb_size);
gst_h264_dpb_set_interlaced (priv->dpb, interlaced);
}
if (!gst_h264_decoder_update_max_num_reorder_frames (self, sps))
return GST_FLOW_ERROR;
return GST_FLOW_OK;
}
static gboolean
gst_h264_decoder_init_gap_picture (GstH264Decoder * self,
GstH264Picture * picture, gint frame_num)
{
picture->nonexisting = TRUE;
picture->nal_ref_idc = 1;
picture->frame_num = picture->pic_num = frame_num;
picture->dec_ref_pic_marking.adaptive_ref_pic_marking_mode_flag = FALSE;
picture->ref = GST_H264_PICTURE_REF_SHORT_TERM;
picture->ref_pic = TRUE;
picture->dec_ref_pic_marking.long_term_reference_flag = FALSE;
picture->field = GST_H264_PICTURE_FIELD_FRAME;
return gst_h264_decoder_calculate_poc (self, picture);
}
static GstFlowReturn
gst_h264_decoder_decode_slice (GstH264Decoder * self)
{
GstH264DecoderClass *klass = GST_H264_DECODER_GET_CLASS (self);
GstH264DecoderPrivate *priv = self->priv;
GstH264Slice *slice = &priv->current_slice;
GstH264Picture *picture = priv->current_picture;
GArray *ref_pic_list0 = NULL;
GArray *ref_pic_list1 = NULL;
GstFlowReturn ret = GST_FLOW_OK;
if (!picture) {
GST_ERROR_OBJECT (self, "No current picture");
return GST_FLOW_ERROR;
}
GST_LOG_OBJECT (self, "Decode picture %p (frame_num %d, poc %d)",
picture, picture->frame_num, picture->pic_order_cnt);
priv->max_pic_num = slice->header.max_pic_num;
if (priv->process_ref_pic_lists) {
if (!gst_h264_decoder_modify_ref_pic_lists (self)) {
ret = GST_FLOW_ERROR;
goto beach;
}
ref_pic_list0 = priv->ref_pic_list0;
ref_pic_list1 = priv->ref_pic_list1;
}
g_assert (klass->decode_slice);
ret = klass->decode_slice (self, picture, slice, ref_pic_list0,
ref_pic_list1);
if (ret != GST_FLOW_OK) {
GST_WARNING_OBJECT (self,
"Subclass didn't want to decode picture %p (frame_num %d, poc %d)",
picture, picture->frame_num, picture->pic_order_cnt);
}
beach:
g_array_set_size (priv->ref_pic_list0, 0);
g_array_set_size (priv->ref_pic_list1, 0);
return ret;
}
static gint
pic_num_desc_compare (const GstH264Picture ** a, const GstH264Picture ** b)
{
return (*b)->pic_num - (*a)->pic_num;
}
static gint
long_term_pic_num_asc_compare (const GstH264Picture ** a,
const GstH264Picture ** b)
{
return (*a)->long_term_pic_num - (*b)->long_term_pic_num;
}
static void
construct_ref_pic_lists_p (GstH264Decoder * self,
GstH264Picture * current_picture)
{
GstH264DecoderPrivate *priv = self->priv;
gint pos;
/* RefPicList0 (8.2.4.2.1) [[1] [2]], where:
* [1] shortterm ref pics sorted by descending pic_num,
* [2] longterm ref pics by ascending long_term_pic_num.
*/
g_array_set_size (priv->ref_pic_list_p0, 0);
gst_h264_dpb_get_pictures_short_term_ref (priv->dpb,
TRUE, FALSE, priv->ref_pic_list_p0);
g_array_sort (priv->ref_pic_list_p0, (GCompareFunc) pic_num_desc_compare);
pos = priv->ref_pic_list_p0->len;
gst_h264_dpb_get_pictures_long_term_ref (priv->dpb,
FALSE, priv->ref_pic_list_p0);
g_qsort_with_data (&g_array_index (priv->ref_pic_list_p0, gpointer, pos),
priv->ref_pic_list_p0->len - pos, sizeof (gpointer),
(GCompareDataFunc) long_term_pic_num_asc_compare, NULL);
#ifndef GST_DISABLE_GST_DEBUG
if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_DEBUG) {
GString *str = g_string_new (NULL);
for (pos = 0; pos < priv->ref_pic_list_p0->len; pos++) {
GstH264Picture *ref =
g_array_index (priv->ref_pic_list_p0, GstH264Picture *, pos);
if (!GST_H264_PICTURE_IS_LONG_TERM_REF (ref))
g_string_append_printf (str, "|%i", ref->pic_num);
else
g_string_append_printf (str, "|%is", ref->pic_num);
}
GST_DEBUG_OBJECT (self, "ref_pic_list_p0: %s|", str->str);
g_string_free (str, TRUE);
}
#endif
}
static gint
frame_num_wrap_desc_compare (const GstH264Picture ** a,
const GstH264Picture ** b)
{
return (*b)->frame_num_wrap - (*a)->frame_num_wrap;
}
static gint
long_term_frame_idx_asc_compare (const GstH264Picture ** a,
const GstH264Picture ** b)
{
return (*a)->long_term_frame_idx - (*b)->long_term_frame_idx;
}
/* init_picture_refs_fields_1 in gstvaapidecoder_h264.c */
static void
init_picture_refs_fields_1 (GstH264Decoder * self, GstH264PictureField field,
GArray * ref_frame_list, GArray * ref_pic_list_x)
{
guint i = 0, j = 0;
do {
for (; i < ref_frame_list->len; i++) {
GstH264Picture *pic = g_array_index (ref_frame_list, GstH264Picture *, i);
if (pic->field == field) {
pic = gst_h264_picture_ref (pic);
g_array_append_val (ref_pic_list_x, pic);
i++;
break;
}
}
for (; j < ref_frame_list->len; j++) {
GstH264Picture *pic = g_array_index (ref_frame_list, GstH264Picture *, j);
if (pic->field != field) {
pic = gst_h264_picture_ref (pic);
g_array_append_val (ref_pic_list_x, pic);
j++;
break;
}
}
} while (i < ref_frame_list->len || j < ref_frame_list->len);
}
static void
construct_ref_field_pic_lists_p (GstH264Decoder * self,
GstH264Picture * current_picture)
{
GstH264DecoderPrivate *priv = self->priv;
gint pos;
g_array_set_size (priv->ref_pic_list_p0, 0);
g_array_set_size (priv->ref_frame_list_0_short_term, 0);
g_array_set_size (priv->ref_frame_list_long_term, 0);
/* 8.2.4.2.2, 8.2.4.2.5 refFrameList0ShortTerm:
* short-term ref pictures sorted by descending frame_num_wrap.
*/
gst_h264_dpb_get_pictures_short_term_ref (priv->dpb,
TRUE, TRUE, priv->ref_frame_list_0_short_term);
g_array_sort (priv->ref_frame_list_0_short_term,
(GCompareFunc) frame_num_wrap_desc_compare);
#ifndef GST_DISABLE_GST_DEBUG
if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE
&& priv->ref_frame_list_0_short_term->len) {
GString *str = g_string_new (NULL);
for (pos = 0; pos < priv->ref_frame_list_0_short_term->len; pos++) {
GstH264Picture *ref = g_array_index (priv->ref_frame_list_0_short_term,
GstH264Picture *, pos);
g_string_append_printf (str, "|%i(%d)", ref->frame_num_wrap, ref->field);
}
GST_TRACE_OBJECT (self, "ref_frame_list_0_short_term (%d): %s|",
current_picture->field, str->str);
g_string_free (str, TRUE);
}
#endif
/* 8.2.4.2.2 refFrameList0LongTerm,:
* long-term ref pictures sorted by ascending long_term_frame_idx.
*/
gst_h264_dpb_get_pictures_long_term_ref (priv->dpb,
TRUE, priv->ref_frame_list_long_term);
g_array_sort (priv->ref_frame_list_long_term,
(GCompareFunc) long_term_frame_idx_asc_compare);
#ifndef GST_DISABLE_GST_DEBUG
if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_TRACE
&& priv->ref_frame_list_long_term->len) {
GString *str = g_string_new (NULL);
for (pos = 0; pos < priv->ref_frame_list_long_term->len; pos++) {
GstH264Picture *ref = g_array_index (priv->ref_frame_list_0_short_term,
GstH264Picture *, pos);
g_string_append_printf (str, "|%i(%d)", ref->long_term_frame_idx,
ref->field);
}
GST_TRACE_OBJECT (self, "ref_frame_list_0_long_term (%d): %s|",
current_picture->field, str->str);
g_string_free (str, TRUE);
}
#endif
/* 8.2.4.2.5 */
init_picture_refs_fields_1 (self, current_picture->field,
priv->ref_frame_list_0_short_term, priv->ref_pic_list_p0);
init_picture_refs_fields_1 (self, current_picture->field,
priv->ref_frame_list_long_term, priv->ref_pic_list_p0);
#ifndef GST_DISABLE_GST_DEBUG
if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) >= GST_LEVEL_DEBUG
&& priv->ref_pic_list_p0->len) {
GString *str = g_string_new (NULL);
for (pos = 0; pos < priv->ref_pic_list_p0->len; pos++) {
GstH264Picture *ref =
g_array_index (priv->ref_pic_list_p0, GstH264Picture *, pos);
if (!GST_H264_PICTURE_IS_LONG_TERM_REF (ref))
g_string_append_printf (str, "|%i(%d)s", ref->frame_num_wrap,
ref->field);
else
g_string_append_printf (str, "|%i(%d)l", ref->long_term_frame_idx,
ref->field);
}
GST_DEBUG_OBJECT (self, "ref_pic_list_p0 (%d): %s|", current_picture->field,
str->str);
g_string_free (str, TRUE);
}
#endif
/* Clear temporary lists, now pictures are owned by ref_pic_list_p0 */
g_array_set_size (priv->ref_frame_list_0_short_term, 0);
g_array_set_size (priv->ref_frame_list_long_term, 0);
}
static gboolean
lists_are_equal (GArray * l1, GArray * l2)
{
gint i;
if (l1->len != l2->len)
return FALSE;
for (i = 0; i < l1->len; i++)
if (g_array_index (l1, gpointer, i) != g_array_index (l2, gpointer, i))
return FALSE;
return TRUE;
}
static gint
split_ref_pic_list_b (GstH264Decoder * self, GArray * ref_pic_list_b,
GCompareFunc compare_func)
{
gint pos;
for (pos = 0; pos < ref_pic_list_b->len; pos++) {
GstH264Picture *pic = g_array_index (ref_pic_list_b, GstH264Picture *, pos);
if (compare_func (&pic, &self->priv->current_picture) > 0)
break;
}
return pos;
}
static void
print_ref_pic_list_b (GstH264Decoder * self, GArray * ref_list_b,
const gchar * name)
{
#ifndef GST_DISABLE_GST_DEBUG
GString *str;
gint i;
if (gst_debug_category_get_threshold (GST_CAT_DEFAULT) < GST_LEVEL_DEBUG)
return;
str = g_string_new (NULL);
for (i = 0; i < ref_list_b->len; i++) {
GstH264Picture *ref = g_array_index (ref_list_b, GstH264Picture *, i);
if (!GST_H264_PICTURE_IS_LONG_TERM_REF (ref))
g_string_append_printf (str, "|%i", ref->pic_order_cnt);
else
g_string_append_printf (str, "|%il", ref->long_term_pic_num);
}
GST_DEBUG_OBJECT (self, "%s: %s| curr %i", name, str->str,
self->priv->current_picture->pic_order_cnt);
g_string_free (str, TRUE);
#endif
}
static void
construct_ref_pic_lists_b (GstH264Decoder * self,
GstH264Picture * current_picture)
{
GstH264DecoderPrivate *priv = self->priv;
gint pos;
/* RefPicList0 (8.2.4.2.3) [[1] [2] [3]], where:
* [1] shortterm ref pics with POC < current_picture's POC sorted by descending POC,
* [2] shortterm ref pics with POC > current_picture's POC by ascending POC,
* [3] longterm ref pics by ascending long_term_pic_num.
*/
g_array_set_size (priv->ref_pic_list_b0, 0);
g_array_set_size (priv->ref_pic_list_b1, 0);
/* 8.2.4.2.3
* When pic_order_cnt_type is equal to 0, reference pictures that are marked
* as "non-existing" as specified in clause 8.2.5.2 are not included in either
* RefPicList0 or RefPicList1
*/
gst_h264_dpb_get_pictures_short_term_ref (priv->dpb,
current_picture->pic_order_cnt_type != 0, FALSE, priv->ref_pic_list_b0);
/* First sort ascending, this will put [1] in right place and finish
* [2]. */
print_ref_pic_list_b (self, priv->ref_pic_list_b0, "ref_pic_list_b0");
g_array_sort (priv->ref_pic_list_b0, (GCompareFunc) poc_asc_compare);
print_ref_pic_list_b (self, priv->ref_pic_list_b0, "ref_pic_list_b0");
/* Find first with POC > current_picture's POC to get first element
* in [2]... */
pos = split_ref_pic_list_b (self, priv->ref_pic_list_b0,
(GCompareFunc) poc_asc_compare);
GST_DEBUG_OBJECT (self, "split point %i", pos);
/* and sort [1] descending, thus finishing sequence [1] [2]. */
g_qsort_with_data (priv->ref_pic_list_b0->data, pos, sizeof (gpointer),
(GCompareDataFunc) poc_desc_compare, NULL);
/* Now add [3] and sort by ascending long_term_pic_num. */
pos = priv->ref_pic_list_b0->len;
gst_h264_dpb_get_pictures_long_term_ref (priv->dpb,
FALSE, priv->ref_pic_list_b0);
g_qsort_with_data (&g_array_index (priv->ref_pic_list_b0, gpointer, pos),
priv->ref_pic_list_b0->len - pos, sizeof (gpointer),
(GCompareDataFunc) long_term_pic_num_asc_compare, NULL);
/* RefPicList1 (8.2.4.2.4) [[1] [2] [3]], where:
* [1] shortterm ref pics with POC > curr_pic's POC sorted by ascending POC,
* [2] shortterm ref pics with POC < curr_pic's POC by descending POC,
* [3] longterm ref pics by ascending long_term_pic_num.
*/
gst_h264_dpb_get_pictures_short_term_ref (priv->dpb,
current_picture->pic_order_cnt_type != 0, FALSE, priv->ref_pic_list_b1);
/* First sort by descending POC. */
g_array_sort (priv->ref_pic_list_b1, (GCompareFunc) poc_desc_compare);
/* Split at first with POC < current_picture's POC to get first element
* in [2]... */
pos = split_ref_pic_list_b (self, priv->ref_pic_list_b1,
(GCompareFunc) poc_desc_compare);
/* and sort [1] ascending. */
g_qsort_with_data (priv->ref_pic_list_b1->data, pos, sizeof (gpointer),
(GCompareDataFunc) poc_asc_compare, NULL);
/* Now add [3] and sort by ascending long_term_pic_num */
pos = priv->ref_pic_list_b1->len;
gst_h264_dpb_get_pictures_long_term_ref (priv->dpb,
FALSE, priv->ref_pic_list_b1);
g_qsort_with_data (&g_array_index (priv->ref_pic_list_b1, gpointer, pos),
priv->ref_pic_list_b1->len - pos, sizeof (gpointer),
(GCompareDataFunc) long_term_pic_num_asc_compare, NULL);
/* If lists identical, swap first two entries in RefPicList1 (spec
* 8.2.4.2.3) */
if (priv->ref_pic_list_b1->len > 1
&& lists_are_equal (priv->ref_pic_list_b0, priv->ref_pic_list_b1)) {
/* swap */
GstH264Picture **list = (GstH264Picture **) priv->ref_pic_list_b1->data;
GstH264Picture *pic = list[0];
list[0] = list[1];
list[1] = pic;
}
print_ref_pic_list_b (self, priv->ref_pic_list_b0, "ref_pic_list_b0");
print_ref_pic_list_b (self, priv->ref_pic_list_b1, "ref_pic_list_b1");
}
static void
construct_ref_field_pic_lists_b (GstH264Decoder * self,
GstH264Picture * current_picture)
{
GstH264DecoderPrivate *priv = self->priv;
gint pos;
/* refFrameList0ShortTerm (8.2.4.2.4) [[1] [2]], where:
* [1] shortterm ref pics with POC < current_picture's POC sorted by descending POC,
* [2] shortterm ref pics with POC > current_picture's POC by ascending POC,
*/
g_array_set_size (priv->ref_pic_list_b0, 0);
g_array_set_size (priv->ref_pic_list_b1, 0);
g_array_set_size (priv->ref_frame_list_0_short_term, 0);
g_array_set_size (priv->ref_frame_list_1_short_term, 0);
g_array_set_size (priv->ref_frame_list_long_term, 0);
/* 8.2.4.2.4
* When pic_order_cnt_type is equal to 0, reference pictures that are marked
* as "non-existing" as specified in clause 8.2.5.2 are not included in either
* RefPicList0 or RefPicList1
*/
gst_h264_dpb_get_pictures_short_term_ref (priv->dpb,
current_picture->pic_order_cnt_type != 0, TRUE,
priv->ref_frame_list_0_short_term);
/* First sort ascending, this will put [1] in right place and finish
* [2]. */
print_ref_pic_list_b (self, priv->ref_frame_list_0_short_term,
"ref_frame_list_0_short_term");
g_array_sort (priv->ref_frame_list_0_short_term,
(GCompareFunc) poc_asc_compare);
print_ref_pic_list_b (self, priv->ref_frame_list_0_short_term,
"ref_frame_list_0_short_term");
/* Find first with POC > current_picture's POC to get first element
* in [2]... */
pos = split_ref_pic_list_b (self, priv->ref_frame_list_0_short_term,
(GCompareFunc) poc_asc_compare);
GST_DEBUG_OBJECT (self, "split point %i", pos);
/* and sort [1] descending, thus finishing sequence [1] [2]. */
g_qsort_with_data (priv->ref_frame_list_0_short_term->data, pos,
sizeof (gpointer), (GCompareDataFunc) poc_desc_compare, NULL);
/* refFrameList1ShortTerm (8.2.4.2.4) [[1] [2]], where:
* [1] shortterm ref pics with POC > curr_pic's POC sorted by ascending POC,
* [2] shortterm ref pics with POC < curr_pic's POC by descending POC,
*/
gst_h264_dpb_get_pictures_short_term_ref (priv->dpb,
current_picture->pic_order_cnt_type != 0, TRUE,
priv->ref_frame_list_1_short_term);
/* First sort by descending POC. */
g_array_sort (priv->ref_frame_list_1_short_term,
(GCompareFunc) poc_desc_compare);
/* Split at first with POC < current_picture's POC to get first element
* in [2]... */
pos = split_ref_pic_list_b (self, priv->ref_frame_list_1_short_term,
(GCompareFunc) poc_desc_compare);
/* and sort [1] ascending. */
g_qsort_with_data (priv->ref_frame_list_1_short_term->data, pos,
sizeof (gpointer), (GCompareDataFunc) poc_asc_compare, NULL);
/* 8.2.4.2.2 refFrameList0LongTerm,:
* long-term ref pictures sorted by ascending long_term_frame_idx.
*/
gst_h264_dpb_get_pictures_long_term_ref (priv->dpb,
TRUE, priv->ref_frame_list_long_term);
g_array_sort (priv->ref_frame_list_long_term,
(GCompareFunc) long_term_frame_idx_asc_compare);
/* 8.2.4.2.5 RefPicList0 */
init_picture_refs_fields_1 (self, current_picture->field,
priv->ref_frame_list_0_short_term, priv->ref_pic_list_b0);
init_picture_refs_fields_1 (self, current_picture->field,
priv->ref_frame_list_long_term, priv->ref_pic_list_b0);
/* 8.2.4.2.5 RefPicList1 */
init_picture_refs_fields_1 (self, current_picture->field,
priv->ref_frame_list_1_short_term, priv->ref_pic_list_b1);
init_picture_refs_fields_1 (self, current_picture->field,
priv->ref_frame_list_long_term, priv->ref_pic_list_b1);
/* If lists identical, swap first two entries in RefPicList1 (spec
* 8.2.4.2.5) */
if (priv->ref_pic_list_b1->len > 1
&& lists_are_equal (priv->ref_pic_list_b0, priv->ref_pic_list_b1)) {
/* swap */
GstH264Picture **list = (GstH264Picture **) priv->ref_pic_list_b1->data;
GstH264Picture *pic = list[0];
list[0] = list[1];
list[1] = pic;
}
print_ref_pic_list_b (self, priv->ref_pic_list_b0, "ref_pic_list_b0");
print_ref_pic_list_b (self, priv->ref_pic_list_b1, "ref_pic_list_b1");
/* Clear temporary lists, now pictures are owned by ref_pic_list_b0
* and ref_pic_list_b1 */
g_array_set_size (priv->ref_frame_list_0_short_term, 0);
g_array_set_size (priv->ref_frame_list_1_short_term, 0);
g_array_set_size (priv->ref_frame_list_long_term, 0);
}
static void
gst_h264_decoder_prepare_ref_pic_lists (GstH264Decoder * self,
GstH264Picture * current_picture)
{
GstH264DecoderPrivate *priv = self->priv;
gboolean construct_list = FALSE;
gint i;
GArray *dpb_array = gst_h264_dpb_get_pictures_all (priv->dpb);
/* 8.2.4.2.1 ~ 8.2.4.2.4
* When this process is invoked, there shall be at least one reference entry
* that is currently marked as "used for reference"
* (i.e., as "used for short-term reference" or "used for long-term reference")
* and is not marked as "non-existing"
*/
for (i = 0; i < dpb_array->len; i++) {
GstH264Picture *picture = g_array_index (dpb_array, GstH264Picture *, i);
if (GST_H264_PICTURE_IS_REF (picture) && !picture->nonexisting) {
construct_list = TRUE;
break;
}
}
g_array_unref (dpb_array);
if (!construct_list) {
gst_h264_decoder_clear_ref_pic_lists (self);
return;
}
if (GST_H264_PICTURE_IS_FRAME (current_picture)) {
construct_ref_pic_lists_p (self, current_picture);
construct_ref_pic_lists_b (self, current_picture);
} else {
construct_ref_field_pic_lists_p (self, current_picture);
construct_ref_field_pic_lists_b (self, current_picture);
}
}
static void
gst_h264_decoder_clear_ref_pic_lists (GstH264Decoder * self)
{
GstH264DecoderPrivate *priv = self->priv;
g_array_set_size (priv->ref_pic_list_p0, 0);
g_array_set_size (priv->ref_pic_list_b0, 0);
g_array_set_size (priv->ref_pic_list_b1, 0);
}
static gint
long_term_pic_num_f (GstH264Decoder * self, const GstH264Picture * picture)
{
if (GST_H264_PICTURE_IS_LONG_TERM_REF (picture))
return picture->long_term_pic_num;
return 2 * (self->priv->max_long_term_frame_idx + 1);
}
static gint
pic_num_f (GstH264Decoder * self, const GstH264Picture * picture)
{
if (!GST_H264_PICTURE_IS_LONG_TERM_REF (picture))
return picture->pic_num;
return self->priv->max_pic_num;
}
/* shift elements on the |array| starting from |from| to |to|,
* inclusive, one position to the right and insert pic at |from| */
static void
shift_right_and_insert (GArray * array, gint from, gint to,
GstH264Picture * picture)
{
g_return_if_fail (from <= to);
g_return_if_fail (array && picture);
g_array_set_size (array, to + 2);
g_array_insert_val (array, from, picture);
}
/* This can process either ref_pic_list0 or ref_pic_list1, depending
* on the list argument. Set up pointers to proper list to be
* processed here. */
static gboolean
modify_ref_pic_list (GstH264Decoder * self, int list)
{
GstH264DecoderPrivate *priv = self->priv;
GstH264Picture *picture = priv->current_picture;
GArray *ref_pic_listx;
const GstH264SliceHdr *slice_hdr = &priv->current_slice.header;
const GstH264RefPicListModification *list_mod;
gboolean ref_pic_list_modification_flag_lX;
gint num_ref_idx_lX_active_minus1;
guint num_ref_pic_list_modifications;
gint i;
gint pic_num_lx_pred = picture->pic_num;
gint ref_idx_lx = 0, src, dst;
gint pic_num_lx_no_wrap;
gint pic_num_lx;
gboolean done = FALSE;
GstH264Picture *pic;
if (list == 0) {
ref_pic_listx = priv->ref_pic_list0;
ref_pic_list_modification_flag_lX =
slice_hdr->ref_pic_list_modification_flag_l0;
num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l0;
num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l0_active_minus1;
list_mod = slice_hdr->ref_pic_list_modification_l0;
} else {
ref_pic_listx = priv->ref_pic_list1;
ref_pic_list_modification_flag_lX =
slice_hdr->ref_pic_list_modification_flag_l1;
num_ref_pic_list_modifications = slice_hdr->n_ref_pic_list_modification_l1;
num_ref_idx_lX_active_minus1 = slice_hdr->num_ref_idx_l1_active_minus1;
list_mod = slice_hdr->ref_pic_list_modification_l1;
}
/* Resize the list to the size requested in the slice header.
*
* Note that per 8.2.4.2 it's possible for
* num_ref_idx_lX_active_minus1 to indicate there should be more ref
* pics on list than we constructed. Those superfluous ones should
* be treated as non-reference and will be initialized to null,
* which must be handled by clients */
g_assert (num_ref_idx_lX_active_minus1 >= 0);
if (ref_pic_listx->len > num_ref_idx_lX_active_minus1 + 1)
g_array_set_size (ref_pic_listx, num_ref_idx_lX_active_minus1 + 1);
if (!ref_pic_list_modification_flag_lX)
return TRUE;
/* Spec 8.2.4.3:
* Reorder pictures on the list in a way specified in the stream. */
for (i = 0; i < num_ref_pic_list_modifications && !done; i++) {
switch (list_mod->modification_of_pic_nums_idc) {
/* 8.2.4.3.1 - Modify short reference picture position. */
case 0:
case 1:
/* 8-34 */
if (list_mod->modification_of_pic_nums_idc == 0) {
/* Substract given value from predicted PicNum. */
pic_num_lx_no_wrap = pic_num_lx_pred -
(list_mod->value.abs_diff_pic_num_minus1 + 1);
/* Wrap around max_pic_num if it becomes < 0 as result of
* subtraction */
if (pic_num_lx_no_wrap < 0)
pic_num_lx_no_wrap += priv->max_pic_num;
} else { /* 8-35 */
/* Add given value to predicted PicNum. */
pic_num_lx_no_wrap = pic_num_lx_pred +
(list_mod->value.abs_diff_pic_num_minus1 + 1);
/* Wrap around max_pic_num if it becomes >= max_pic_num as
* result of the addition */
if (pic_num_lx_no_wrap >= priv->max_pic_num)
pic_num_lx_no_wrap -= priv->max_pic_num;
}
/* For use in next iteration */
pic_num_lx_pred = pic_num_lx_no_wrap;
/* 8-36 */
if (pic_num_lx_no_wrap > picture->pic_num)
pic_num_lx = pic_num_lx_no_wrap - priv->max_pic_num;
else
pic_num_lx = pic_num_lx_no_wrap;
/* 8-37 */
g_assert (num_ref_idx_lX_active_minus1 + 1 < 32);
pic = gst_h264_dpb_get_short_ref_by_pic_num (priv->dpb, pic_num_lx);
if (!pic) {
GST_WARNING_OBJECT (self, "Malformed stream, no pic num %d",
pic_num_lx);
break;
}
shift_right_and_insert (ref_pic_listx, ref_idx_lx,
num_ref_idx_lX_active_minus1, pic);
ref_idx_lx++;
for (src = ref_idx_lx, dst = ref_idx_lx;
src <= num_ref_idx_lX_active_minus1 + 1; src++) {
GstH264Picture *src_pic =
g_array_index (ref_pic_listx, GstH264Picture *, src);
gint src_pic_num_lx = src_pic ? pic_num_f (self, src_pic) : -1;
if (src_pic_num_lx != pic_num_lx)
g_array_index (ref_pic_listx, GstH264Picture *, dst++) = src_pic;
}
break;
/* 8.2.4.3.2 - Long-term reference pictures */
case 2:
/* (8-28) */
g_assert (num_ref_idx_lX_active_minus1 + 1 < 32);
pic = gst_h264_dpb_get_long_ref_by_long_term_pic_num (priv->dpb,
list_mod->value.long_term_pic_num);
if (!pic) {
GST_WARNING_OBJECT (self, "Malformed stream, no pic num %d",
list_mod->value.long_term_pic_num);
break;
}
shift_right_and_insert (ref_pic_listx, ref_idx_lx,
num_ref_idx_lX_active_minus1, pic);
ref_idx_lx++;
for (src = ref_idx_lx, dst = ref_idx_lx;
src <= num_ref_idx_lX_active_minus1 + 1; src++) {
GstH264Picture *src_pic =
g_array_index (ref_pic_listx, GstH264Picture *, src);
if (long_term_pic_num_f (self, src_pic) !=
list_mod->value.long_term_pic_num)
g_array_index (ref_pic_listx, GstH264Picture *, dst++) = src_pic;
}
break;
/* End of modification list */
case 3:
done = TRUE;
break;
default:
/* may be recoverable */
GST_WARNING ("Invalid modification_of_pic_nums_idc = %d",
list_mod->modification_of_pic_nums_idc);
break;
}
list_mod++;
}
/* Per NOTE 2 in 8.2.4.3.2, the ref_pic_listx in the above loop is
* temporarily made one element longer than the required final list.
* Resize the list back to its required size. */
if (ref_pic_listx->len > num_ref_idx_lX_active_minus1 + 1)
g_array_set_size (ref_pic_listx, num_ref_idx_lX_active_minus1 + 1);
return TRUE;
}
static void
copy_pic_list_into (GArray * dest, GArray * src)
{
gint i;
g_array_set_size (dest, 0);
for (i = 0; i < src->len; i++)
g_array_append_val (dest, g_array_index (src, gpointer, i));
}
static gboolean
gst_h264_decoder_modify_ref_pic_lists (GstH264Decoder * self)
{
GstH264DecoderPrivate *priv = self->priv;
GstH264SliceHdr *slice_hdr = &priv->current_slice.header;
g_array_set_size (priv->ref_pic_list0, 0);
g_array_set_size (priv->ref_pic_list1, 0);
if (GST_H264_IS_P_SLICE (slice_hdr) || GST_H264_IS_SP_SLICE (slice_hdr)) {
/* 8.2.4 fill reference picture list RefPicList0 for P or SP slice */
copy_pic_list_into (priv->ref_pic_list0, priv->ref_pic_list_p0);
return modify_ref_pic_list (self, 0);
} else if (GST_H264_IS_B_SLICE (slice_hdr)) {
/* 8.2.4 fill reference picture list RefPicList0 and RefPicList1 for B slice */
copy_pic_list_into (priv->ref_pic_list0, priv->ref_pic_list_b0);
copy_pic_list_into (priv->ref_pic_list1, priv->ref_pic_list_b1);
return modify_ref_pic_list (self, 0)
&& modify_ref_pic_list (self, 1);
}
return TRUE;
}
/**
* gst_h264_decoder_set_process_ref_pic_lists:
* @decoder: a #GstH264Decoder
* @process: whether subclass is requiring reference picture modification process
*
* Called to en/disable reference picture modification process.
*
* Since: 1.18
*/
void
gst_h264_decoder_set_process_ref_pic_lists (GstH264Decoder * decoder,
gboolean process)
{
decoder->priv->process_ref_pic_lists = process;
}
/**
* gst_h264_decoder_get_picture:
* @decoder: a #GstH264Decoder
* @system_frame_number: a target system frame number of #GstH264Picture
*
* Retrive DPB and return a #GstH264Picture corresponding to
* the @system_frame_number
*
* Returns: (transfer full): a #GstH264Picture if successful, or %NULL otherwise
*
* Since: 1.18
*/
GstH264Picture *
gst_h264_decoder_get_picture (GstH264Decoder * decoder,
guint32 system_frame_number)
{
return gst_h264_dpb_get_picture (decoder->priv->dpb, system_frame_number);
}
Reference in a new issue View git blame Copy permalink