mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-12-12 03:16:33 +00:00
Code Issues 1 Projects Releases Wiki Activity
gstreamer/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c
Matthew Waters 86f00367fc androidmedia/enc: handle codec-data before popping GstVideoCodecFrames 
Issue is that when amc was producing a codec-data buffer, a
GstVideoCodecFrame was being popped off the internal queue.  This meant
that the codec-data was being associated with the first input frame and
the second (first encoded buffer) output buffer with the second input
frame.  At the end (assuming one input produces one output which seems
to hold in my testing and how the encoder is currently implemented)
there would be an input frame missing and would be pushed without any
timing information.  This would lead to e.g. muxers rejecting the buffer
without PTS and failing to mux.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5330>
2023-09-15 17:47:17 +03:00

1837 lines
54 KiB
C
Raw Blame History

/*
* Initially based on gst-plugins-bad/sys/androidmedia/gstamcvideodec.c
*
* Copyright (C) 2011, Hewlett-Packard Development Company, L.P.
* Author: Sebastian Dröge <sebastian.droege@collabora.co.uk>, Collabora Ltd.
*
* Copyright (C) 2012, Collabora Ltd.
* Author: Sebastian Dröge <sebastian.droege@collabora.co.uk>
*
* Copyright (C) 2013, Lemote Ltd.
* Author: Chen Jie <chenj@lemote.com>
*
* Copyright (C) 2015, Sebastian Dröge <sebastian@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation
* version 2.1 of the License.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <gst/gst.h>
#include <string.h>
#ifdef HAVE_ORC
#include <orc/orc.h>
#else
#define orc_memcpy memcpy
#endif
#ifdef HAVE_JNI_H
#include "gstjniutils.h"
#endif
#include "gstamcvideoenc.h"
#include "gstamc-constants.h"
GST_DEBUG_CATEGORY_STATIC (gst_amc_video_enc_debug_category);
#define GST_CAT_DEFAULT gst_amc_video_enc_debug_category
typedef struct _BufferIdentification BufferIdentification;
struct _BufferIdentification
{
guint64 timestamp;
};
static BufferIdentification *
buffer_identification_new (GstClockTime timestamp)
{
BufferIdentification *id = g_slice_new (BufferIdentification);
id->timestamp = timestamp;
return id;
}
static void
buffer_identification_free (BufferIdentification * id)
{
g_slice_free (BufferIdentification, id);
}
/* prototypes */
static void gst_amc_video_enc_finalize (GObject * object);
static GstStateChangeReturn
gst_amc_video_enc_change_state (GstElement * element,
GstStateChange transition);
static gboolean gst_amc_video_enc_open (GstVideoEncoder * encoder);
static gboolean gst_amc_video_enc_close (GstVideoEncoder * encoder);
static gboolean gst_amc_video_enc_start (GstVideoEncoder * encoder);
static gboolean gst_amc_video_enc_stop (GstVideoEncoder * encoder);
static gboolean gst_amc_video_enc_set_format (GstVideoEncoder * encoder,
GstVideoCodecState * state);
static gboolean gst_amc_video_enc_flush (GstVideoEncoder * encoder);
static GstFlowReturn gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder,
GstVideoCodecFrame * frame);
static GstFlowReturn gst_amc_video_enc_finish (GstVideoEncoder * encoder);
static GstFlowReturn gst_amc_video_enc_drain (GstAmcVideoEnc * self);
#define BIT_RATE_DEFAULT (2 * 1024 * 1024)
#define I_FRAME_INTERVAL_DEFAULT 0
enum
{
PROP_0,
PROP_BIT_RATE,
PROP_I_FRAME_INTERVAL,
PROP_I_FRAME_INTERVAL_FLOAT
};
/* class initialization */
static void gst_amc_video_enc_class_init (GstAmcVideoEncClass * klass);
static void gst_amc_video_enc_init (GstAmcVideoEnc * self);
static void gst_amc_video_enc_base_init (gpointer g_class);
static GstVideoEncoderClass *parent_class = NULL;
GType
gst_amc_video_enc_get_type (void)
{
static gsize type = 0;
if (g_once_init_enter (&type)) {
GType _type;
static const GTypeInfo info = {
sizeof (GstAmcVideoEncClass),
gst_amc_video_enc_base_init,
NULL,
(GClassInitFunc) gst_amc_video_enc_class_init,
NULL,
NULL,
sizeof (GstAmcVideoEnc),
0,
(GInstanceInitFunc) gst_amc_video_enc_init,
NULL
};
_type = g_type_register_static (GST_TYPE_VIDEO_ENCODER, "GstAmcVideoEnc",
&info, 0);
GST_DEBUG_CATEGORY_INIT (gst_amc_video_enc_debug_category, "amcvideoenc", 0,
"Android MediaCodec video encoder");
g_once_init_leave (&type, _type);
}
return type;
}
static GstAmcFormat *
create_amc_format (GstAmcVideoEnc * encoder, GstVideoCodecState * input_state,
GstCaps * src_caps)
{
GstAmcVideoEncClass *klass;
GstStructure *s;
const gchar *name;
const gchar *mime = NULL;
const gchar *profile_string = NULL;
const gchar *level_string = NULL;
struct
{
const gchar *key;
gint id;
} amc_profile = {
NULL, -1
};
struct
{
const gchar *key;
gint id;
} amc_level = {
NULL, -1
};
gint color_format;
gint stride, slice_height;
GstAmcFormat *format = NULL;
GstVideoInfo *info = &input_state->info;
GError *err = NULL;
klass = GST_AMC_VIDEO_ENC_GET_CLASS (encoder);
s = gst_caps_get_structure (src_caps, 0);
if (!s)
return NULL;
name = gst_structure_get_name (s);
profile_string = gst_structure_get_string (s, "profile");
level_string = gst_structure_get_string (s, "level");
if (strcmp (name, "video/mpeg") == 0) {
gint mpegversion;
if (!gst_structure_get_int (s, "mpegversion", &mpegversion))
return NULL;
if (mpegversion == 4) {
mime = "video/mp4v-es";
if (profile_string) {
amc_profile.key = "profile"; /* named profile ? */
amc_profile.id = gst_amc_mpeg4_profile_from_string (profile_string);
}
if (level_string) {
amc_level.key = "level"; /* named level ? */
amc_level.id = gst_amc_mpeg4_level_from_string (level_string);
}
} else if ( /* mpegversion == 1 || */ mpegversion == 2)
mime = "video/mpeg2";
} else if (strcmp (name, "video/x-h263") == 0) {
mime = "video/3gpp";
} else if (strcmp (name, "video/x-h264") == 0) {
mime = "video/avc";
if (profile_string) {
amc_profile.key = "profile"; /* named profile ? */
amc_profile.id = gst_amc_avc_profile_from_string (profile_string);
}
if (level_string) {
amc_level.key = "level"; /* named level ? */
amc_level.id = gst_amc_avc_level_from_string (level_string);
}
} else if (strcmp (name, "video/x-h265") == 0) {
const gchar *tier_string = gst_structure_get_string (s, "tier");
mime = "video/hevc";
if (profile_string) {
amc_profile.key = "profile"; /* named profile ? */
amc_profile.id = gst_amc_hevc_profile_from_string (profile_string);
}
if (level_string && tier_string) {
amc_level.key = "level"; /* named level ? */
amc_level.id =
gst_amc_hevc_tier_level_from_string (tier_string, level_string);
}
} else if (strcmp (name, "video/x-vp8") == 0) {
mime = "video/x-vnd.on2.vp8";
} else if (strcmp (name, "video/x-vp9") == 0) {
mime = "video/x-vnd.on2.vp9";
} else if (strcmp (name, "video/x-av1") == 0) {
mime = "video/av01";
} else {
GST_ERROR_OBJECT (encoder, "Failed to convert caps(%s/...) to any mime",
name);
return NULL;
}
format = gst_amc_format_new_video (mime, info->width, info->height, &err);
if (!format) {
GST_ERROR_OBJECT (encoder, "Failed to create a \"%s,%dx%d\" MediaFormat",
mime, info->width, info->height);
GST_ELEMENT_ERROR_FROM_ERROR (encoder, err);
return NULL;
}
color_format =
gst_amc_video_format_to_color_format (klass->codec_info,
mime, info->finfo->format);
if (color_format == -1)
goto video_format_failed_to_convert;
gst_amc_format_set_int (format, "bitrate", encoder->bitrate, &err);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (encoder, err);
gst_amc_format_set_int (format, "color-format", color_format, &err);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (encoder, err);
stride = GST_ROUND_UP_4 (info->width); /* safe (?) */
gst_amc_format_set_int (format, "stride", stride, &err);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (encoder, err);
slice_height = info->height;
gst_amc_format_set_int (format, "slice-height", slice_height, &err);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (encoder, err);
if (profile_string) {
if (amc_profile.id == -1)
goto unsupported_profile;
/* FIXME: Set to any value in AVCProfile* leads to
* codec configuration fail */
/* gst_amc_format_set_int (format, amc_profile.key, 0x40); */
}
if (level_string) {
if (amc_level.id == -1)
goto unsupported_level;
/* gst_amc_format_set_int (format, amc_level.key, amc_level.id); */
}
/* On Android N_MR1 and higher, i-frame-interval can be a float value */
#ifdef HAVE_JNI_H
if (gst_amc_jni_get_android_level () >= 25) {
GST_LOG_OBJECT (encoder, "Setting i-frame-interval to %f",
encoder->i_frame_int);
gst_amc_format_set_float (format, "i-frame-interval", encoder->i_frame_int,
&err);
} else
#endif
{
int i_frame_int = encoder->i_frame_int;
/* Round a fractional interval to 1 per sec on older Android */
if (encoder->i_frame_int > 0 && encoder->i_frame_int < 1.0)
i_frame_int = 1;
gst_amc_format_set_int (format, "i-frame-interval", i_frame_int, &err);
}
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (encoder, err);
if (info->fps_d)
gst_amc_format_set_float (format, "frame-rate",
((gfloat) info->fps_n) / info->fps_d, &err);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (encoder, err);
encoder->format = info->finfo->format;
if (!gst_amc_color_format_info_set (&encoder->color_format_info,
klass->codec_info, mime, color_format, info->width, info->height,
stride, slice_height, 0, 0, 0, 0))
goto color_format_info_failed_to_set;
GST_DEBUG_OBJECT (encoder,
"Color format info: {color_format=%d, width=%d, height=%d, "
"stride=%d, slice-height=%d, crop-left=%d, crop-top=%d, "
"crop-right=%d, crop-bottom=%d, frame-size=%d}",
encoder->color_format_info.color_format, encoder->color_format_info.width,
encoder->color_format_info.height, encoder->color_format_info.stride,
encoder->color_format_info.slice_height,
encoder->color_format_info.crop_left, encoder->color_format_info.crop_top,
encoder->color_format_info.crop_right,
encoder->color_format_info.crop_bottom,
encoder->color_format_info.frame_size);
return format;
video_format_failed_to_convert:
GST_ERROR_OBJECT (encoder, "Failed to convert video format");
gst_amc_format_free (format);
return NULL;
color_format_info_failed_to_set:
GST_ERROR_OBJECT (encoder, "Failed to set up GstAmcColorFormatInfo");
gst_amc_format_free (format);
return NULL;
unsupported_profile:
GST_ERROR_OBJECT (encoder, "Unsupported profile '%s'", profile_string);
gst_amc_format_free (format);
return NULL;
unsupported_level:
GST_ERROR_OBJECT (encoder, "Unsupported level '%s'", level_string);
gst_amc_format_free (format);
return NULL;
}
static GstCaps *
caps_from_amc_format (GstAmcFormat * amc_format)
{
GstCaps *caps = NULL;
gchar *mime = NULL;
gint width, height;
gint amc_profile, amc_level;
gfloat frame_rate = 0.0;
gint fraction_n, fraction_d;
GError *err = NULL;
if (!gst_amc_format_get_string (amc_format, "mime", &mime, &err)) {
GST_ERROR ("Failed to get 'mime': %s", err->message);
g_clear_error (&err);
return NULL;
}
if (!gst_amc_format_get_int (amc_format, "width", &width, &err) ||
!gst_amc_format_get_int (amc_format, "height", &height, &err)) {
GST_ERROR ("Failed to get size: %s", err->message);
g_clear_error (&err);
g_free (mime);
return NULL;
}
gst_amc_format_get_float (amc_format, "frame-rate", &frame_rate, NULL);
gst_util_double_to_fraction (frame_rate, &fraction_n, &fraction_d);
if (strcmp (mime, "video/mp4v-es") == 0) {
const gchar *profile_string, *level_string;
caps =
gst_caps_new_simple ("video/mpeg", "mpegversion", G_TYPE_INT, 4,
"systemstream", G_TYPE_BOOLEAN, FALSE, NULL);
if (gst_amc_format_get_int (amc_format, "profile", &amc_profile, NULL)) {
profile_string = gst_amc_mpeg4_profile_to_string (amc_profile);
if (!profile_string)
goto unsupported_profile;
gst_caps_set_simple (caps, "profile", G_TYPE_STRING, profile_string,
NULL);
}
if (gst_amc_format_get_int (amc_format, "level", &amc_level, NULL)) {
level_string = gst_amc_mpeg4_level_to_string (amc_profile);
if (!level_string)
goto unsupported_level;
gst_caps_set_simple (caps, "level", G_TYPE_STRING, level_string, NULL);
}
} else if (strcmp (mime, "video/mpeg2") == 0) {
caps = gst_caps_new_simple ("video/mpeg", "mpegversion", 2, NULL);
} else if (strcmp (mime, "video/3gpp") == 0) {
caps = gst_caps_new_empty_simple ("video/x-h263");
} else if (strcmp (mime, "video/avc") == 0) {
const gchar *profile_string, *level_string;
caps =
gst_caps_new_simple ("video/x-h264",
"stream-format", G_TYPE_STRING, "byte-stream", NULL);
if (gst_amc_format_get_int (amc_format, "profile", &amc_profile, NULL)) {
profile_string = gst_amc_avc_profile_to_string (amc_profile, NULL);
if (!profile_string)
goto unsupported_profile;
gst_caps_set_simple (caps, "profile", G_TYPE_STRING, profile_string,
NULL);
}
if (gst_amc_format_get_int (amc_format, "level", &amc_level, NULL)) {
level_string = gst_amc_avc_level_to_string (amc_profile);
if (!level_string)
goto unsupported_level;
gst_caps_set_simple (caps, "level", G_TYPE_STRING, level_string, NULL);
}
} else if (strcmp (mime, "video/hevc") == 0) {
const gchar *profile_string, *level_string, *tier_string;
caps =
gst_caps_new_simple ("video/x-h265",
"stream-format", G_TYPE_STRING, "byte-stream", NULL);
if (gst_amc_format_get_int (amc_format, "profile", &amc_profile, NULL)) {
profile_string = gst_amc_avc_profile_to_string (amc_profile, NULL);
if (!profile_string)
goto unsupported_profile;
gst_caps_set_simple (caps, "profile", G_TYPE_STRING, profile_string,
NULL);
}
if (gst_amc_format_get_int (amc_format, "level", &amc_level, NULL)) {
level_string =
gst_amc_hevc_tier_level_to_string (amc_profile, &tier_string);
if (!level_string || !tier_string)
goto unsupported_level;
gst_caps_set_simple (caps,
"level", G_TYPE_STRING, level_string,
"tier", G_TYPE_STRING, tier_string, NULL);
}
} else if (strcmp (mime, "video/x-vnd.on2.vp8") == 0) {
caps = gst_caps_new_empty_simple ("video/x-vp8");
} else if (strcmp (mime, "video/x-vnd.on2.vp9") == 0) {
caps = gst_caps_new_empty_simple ("video/x-vp9");
} else if (strcmp (mime, "video/av01") == 0) {
caps = gst_caps_new_simple ("video/x-av1",
"stream-format", G_TYPE_STRING, "obu-stream",
"alignment", G_TYPE_STRING, "tu", NULL);
}
gst_caps_set_simple (caps, "width", G_TYPE_INT, width,
"height", G_TYPE_INT, height,
"framerate", GST_TYPE_FRACTION, fraction_n, fraction_d, NULL);
g_free (mime);
return caps;
unsupported_profile:
GST_ERROR ("Unsupported amc profile id %d", amc_profile);
g_free (mime);
gst_caps_unref (caps);
return NULL;
unsupported_level:
GST_ERROR ("Unsupported amc level id %d", amc_level);
g_free (mime);
gst_caps_unref (caps);
return NULL;
}
static void
gst_amc_video_enc_base_init (gpointer g_class)
{
GstElementClass *element_class = GST_ELEMENT_CLASS (g_class);
GstAmcVideoEncClass *videoenc_class = GST_AMC_VIDEO_ENC_CLASS (g_class);
const GstAmcCodecInfo *codec_info;
GstPadTemplate *templ;
GstCaps *sink_caps, *src_caps;
gchar *longname;
codec_info =
g_type_get_qdata (G_TYPE_FROM_CLASS (g_class), gst_amc_codec_info_quark);
/* This happens for the base class and abstract subclasses */
if (!codec_info)
return;
videoenc_class->codec_info = codec_info;
gst_amc_codec_info_to_caps (codec_info, &sink_caps, &src_caps);
/* Add pad templates */
templ =
gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, sink_caps);
gst_element_class_add_pad_template (element_class, templ);
gst_caps_unref (sink_caps);
templ = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, src_caps);
gst_element_class_add_pad_template (element_class, templ);
gst_caps_unref (src_caps);
longname = g_strdup_printf ("Android MediaCodec %s", codec_info->name);
gst_element_class_set_metadata (element_class,
codec_info->name,
"Codec/Encoder/Video/Hardware",
longname, "Sebastian Dröge <sebastian.droege@collabora.co.uk>");
g_free (longname);
}
static void
gst_amc_video_enc_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstAmcVideoEnc *encoder;
GstState state;
gboolean codec_active;
GError *err = NULL;
encoder = GST_AMC_VIDEO_ENC (object);
GST_OBJECT_LOCK (encoder);
state = GST_STATE (encoder);
codec_active = (encoder->codec && state != GST_STATE_READY
&& state != GST_STATE_NULL);
switch (prop_id) {
case PROP_BIT_RATE:
encoder->bitrate = g_value_get_uint (value);
g_mutex_lock (&encoder->codec_lock);
if (encoder->codec) {
if (!gst_amc_codec_set_dynamic_bitrate (encoder->codec, &err,
encoder->bitrate)) {
g_mutex_unlock (&encoder->codec_lock);
goto wrong_state;
}
}
g_mutex_unlock (&encoder->codec_lock);
if (err) {
GST_ELEMENT_WARNING_FROM_ERROR (encoder, err);
g_clear_error (&err);
}
break;
case PROP_I_FRAME_INTERVAL:
encoder->i_frame_int = g_value_get_uint (value);
if (codec_active)
goto wrong_state;
break;
case PROP_I_FRAME_INTERVAL_FLOAT:
encoder->i_frame_int = g_value_get_float (value);
if (codec_active)
goto wrong_state;
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
GST_OBJECT_UNLOCK (encoder);
return;
/* ERROR */
wrong_state:
{
GST_WARNING_OBJECT (encoder, "setting property in wrong state");
GST_OBJECT_UNLOCK (encoder);
}
}
static void
gst_amc_video_enc_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstAmcVideoEnc *encoder;
encoder = GST_AMC_VIDEO_ENC (object);
GST_OBJECT_LOCK (encoder);
switch (prop_id) {
case PROP_BIT_RATE:
g_value_set_uint (value, encoder->bitrate);
break;
case PROP_I_FRAME_INTERVAL:
g_value_set_uint (value, encoder->i_frame_int);
break;
case PROP_I_FRAME_INTERVAL_FLOAT:
g_value_set_float (value, encoder->i_frame_int);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
GST_OBJECT_UNLOCK (encoder);
}
static void
gst_amc_video_enc_class_init (GstAmcVideoEncClass * klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
GstVideoEncoderClass *videoenc_class = GST_VIDEO_ENCODER_CLASS (klass);
GParamFlags dynamic_flag = 0;
parent_class = g_type_class_peek_parent (klass);
gobject_class->set_property = gst_amc_video_enc_set_property;
gobject_class->get_property = gst_amc_video_enc_get_property;
gobject_class->finalize = gst_amc_video_enc_finalize;
element_class->change_state =
GST_DEBUG_FUNCPTR (gst_amc_video_enc_change_state);
videoenc_class->start = GST_DEBUG_FUNCPTR (gst_amc_video_enc_start);
videoenc_class->stop = GST_DEBUG_FUNCPTR (gst_amc_video_enc_stop);
videoenc_class->open = GST_DEBUG_FUNCPTR (gst_amc_video_enc_open);
videoenc_class->close = GST_DEBUG_FUNCPTR (gst_amc_video_enc_close);
videoenc_class->flush = GST_DEBUG_FUNCPTR (gst_amc_video_enc_flush);
videoenc_class->set_format = GST_DEBUG_FUNCPTR (gst_amc_video_enc_set_format);
videoenc_class->handle_frame =
GST_DEBUG_FUNCPTR (gst_amc_video_enc_handle_frame);
videoenc_class->finish = GST_DEBUG_FUNCPTR (gst_amc_video_enc_finish);
// On Android >= 19, we can set bitrate dynamically
// so add the flag so apps can detect it.
if (gst_amc_codec_have_dynamic_bitrate ())
dynamic_flag = GST_PARAM_MUTABLE_PLAYING;
g_object_class_install_property (gobject_class, PROP_BIT_RATE,
g_param_spec_uint ("bitrate", "Bitrate", "Bitrate in bit/sec", 1,
G_MAXINT, BIT_RATE_DEFAULT,
dynamic_flag | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_I_FRAME_INTERVAL,
g_param_spec_uint ("i-frame-interval", "I-frame interval",
"The frequency of I frames expressed in seconds between I frames (0 for automatic)",
0, G_MAXINT, I_FRAME_INTERVAL_DEFAULT,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_I_FRAME_INTERVAL_FLOAT,
g_param_spec_float ("i-frame-interval-float", "I-frame interval",
"The frequency of I frames expressed in seconds between I frames (0 for automatic). "
"Fractional intervals work on Android >= 25",
0, G_MAXFLOAT, I_FRAME_INTERVAL_DEFAULT,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
}
static void
gst_amc_video_enc_init (GstAmcVideoEnc * self)
{
g_mutex_init (&self->codec_lock);
g_mutex_init (&self->drain_lock);
g_cond_init (&self->drain_cond);
self->bitrate = BIT_RATE_DEFAULT;
self->i_frame_int = I_FRAME_INTERVAL_DEFAULT;
}
static gboolean
gst_amc_video_enc_open (GstVideoEncoder * encoder)
{
GstAmcVideoEnc *self = GST_AMC_VIDEO_ENC (encoder);
GstAmcVideoEncClass *klass = GST_AMC_VIDEO_ENC_GET_CLASS (self);
GError *err = NULL;
GST_DEBUG_OBJECT (self, "Opening encoder");
g_mutex_lock (&self->codec_lock);
self->codec = gst_amc_codec_new (klass->codec_info->name, TRUE, &err);
if (!self->codec) {
g_mutex_unlock (&self->codec_lock);
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
return FALSE;
}
g_mutex_unlock (&self->codec_lock);
self->started = FALSE;
self->flushing = TRUE;
GST_DEBUG_OBJECT (self, "Opened encoder");
return TRUE;
}
static gboolean
gst_amc_video_enc_close (GstVideoEncoder * encoder)
{
GstAmcVideoEnc *self = GST_AMC_VIDEO_ENC (encoder);
GST_DEBUG_OBJECT (self, "Closing encoder");
g_mutex_lock (&self->codec_lock);
if (self->codec) {
GError *err = NULL;
gst_amc_codec_release (self->codec, &err);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
gst_amc_codec_free (self->codec);
}
self->codec = NULL;
g_mutex_unlock (&self->codec_lock);
self->started = FALSE;
self->flushing = TRUE;
GST_DEBUG_OBJECT (self, "Closed encoder");
return TRUE;
}
static void
gst_amc_video_enc_finalize (GObject * object)
{
GstAmcVideoEnc *self = GST_AMC_VIDEO_ENC (object);
g_mutex_clear (&self->codec_lock);
g_mutex_clear (&self->drain_lock);
g_cond_clear (&self->drain_cond);
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static GstStateChangeReturn
gst_amc_video_enc_change_state (GstElement * element, GstStateChange transition)
{
GstAmcVideoEnc *self;
GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
GError *err = NULL;
g_return_val_if_fail (GST_IS_AMC_VIDEO_ENC (element),
GST_STATE_CHANGE_FAILURE);
self = GST_AMC_VIDEO_ENC (element);
switch (transition) {
case GST_STATE_CHANGE_NULL_TO_READY:
break;
case GST_STATE_CHANGE_READY_TO_PAUSED:
self->downstream_flow_ret = GST_FLOW_OK;
self->draining = FALSE;
self->started = FALSE;
break;
case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
break;
case GST_STATE_CHANGE_PAUSED_TO_READY:
self->flushing = TRUE;
gst_amc_codec_flush (self->codec, &err);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
g_mutex_lock (&self->drain_lock);
self->draining = FALSE;
g_cond_broadcast (&self->drain_cond);
g_mutex_unlock (&self->drain_lock);
break;
default:
break;
}
if (ret == GST_STATE_CHANGE_FAILURE)
return ret;
ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
if (ret == GST_STATE_CHANGE_FAILURE)
return ret;
switch (transition) {
case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
break;
case GST_STATE_CHANGE_PAUSED_TO_READY:
self->downstream_flow_ret = GST_FLOW_FLUSHING;
self->started = FALSE;
break;
case GST_STATE_CHANGE_READY_TO_NULL:
break;
default:
break;
}
return ret;
}
#define MAX_FRAME_DIST_TIME (5 * GST_SECOND)
#define MAX_FRAME_DIST_FRAMES (100)
static GstVideoCodecFrame *
_find_nearest_frame (GstAmcVideoEnc * self, GstClockTime reference_timestamp)
{
GList *l, *best_l = NULL;
GList *finish_frames = NULL;
GstVideoCodecFrame *best = NULL;
guint64 best_timestamp = 0;
guint64 best_diff = G_MAXUINT64;
BufferIdentification *best_id = NULL;
GList *frames;
frames = gst_video_encoder_get_frames (GST_VIDEO_ENCODER (self));
for (l = frames; l; l = l->next) {
GstVideoCodecFrame *tmp = l->data;
BufferIdentification *id = gst_video_codec_frame_get_user_data (tmp);
guint64 timestamp, diff;
/* This happens for frames that were just added but
* which were not passed to the component yet. Ignore
* them here!
*/
if (!id)
continue;
timestamp = id->timestamp;
if (timestamp > reference_timestamp)
diff = timestamp - reference_timestamp;
else
diff = reference_timestamp - timestamp;
if (best == NULL || diff < best_diff) {
best = tmp;
best_timestamp = timestamp;
best_diff = diff;
best_l = l;
best_id = id;
/* For frames without timestamp we simply take the first frame */
if ((reference_timestamp == 0 && !GST_CLOCK_TIME_IS_VALID (timestamp))
|| diff == 0)
break;
}
}
if (best_id) {
for (l = frames; l && l != best_l; l = l->next) {
GstVideoCodecFrame *tmp = l->data;
BufferIdentification *id = gst_video_codec_frame_get_user_data (tmp);
guint64 diff_time, diff_frames;
if (id->timestamp > best_timestamp)
break;
if (id->timestamp == 0 || best_timestamp == 0)
diff_time = 0;
else
diff_time = best_timestamp - id->timestamp;
diff_frames = best->system_frame_number - tmp->system_frame_number;
if (diff_time > MAX_FRAME_DIST_TIME
|| diff_frames > MAX_FRAME_DIST_FRAMES) {
finish_frames =
g_list_prepend (finish_frames, gst_video_codec_frame_ref (tmp));
}
}
}
if (finish_frames) {
g_warning ("%s: Too old frames, bug in encoder -- please file a bug",
GST_ELEMENT_NAME (self));
for (l = finish_frames; l; l = l->next) {
gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (self), l->data);
}
}
if (best)
gst_video_codec_frame_ref (best);
GST_DEBUG_OBJECT (self, "found best %p from %u frames", best,
g_list_length (frames));
if (best) {
GST_LOG_OBJECT (self, "best %p (input pts %" GST_TIME_FORMAT " dts %"
GST_TIME_FORMAT " frame no %" G_GUINT32_FORMAT " buffer %"
GST_PTR_FORMAT, best, GST_TIME_ARGS (best->pts),
GST_TIME_ARGS (best->dts), best->system_frame_number,
best->input_buffer);
}
g_list_foreach (frames, (GFunc) gst_video_codec_frame_unref, NULL);
g_list_free (frames);
return best;
}
static gboolean
gst_amc_video_enc_set_src_caps (GstAmcVideoEnc * self, GstAmcFormat * format)
{
GstCaps *caps;
GstVideoCodecState *output_state;
GstStructure *s;
caps = caps_from_amc_format (format);
if (!caps) {
GST_ERROR_OBJECT (self, "Failed to create output caps");
return FALSE;
}
/* It may not be proper to reference self->input_state here,
* because MediaCodec is an async model -- input_state may change multiple times,
* the passed-in MediaFormat may not be the one matched to the current input_state.
*
* Though, currently, the final src caps only calculate
* width/height/pixel-aspect-ratio/framerate/codec_data from self->input_state.
*
* If input width/height/codec_data change(is_format_change), it will restart
* MediaCodec, which means in these cases, self->input_state is matched.
*/
output_state = gst_video_encoder_set_output_state (GST_VIDEO_ENCODER (self),
caps, self->input_state);
gst_video_codec_state_unref (output_state);
if (!gst_video_encoder_negotiate (GST_VIDEO_ENCODER (self)))
return FALSE;
output_state = gst_video_encoder_get_output_state (GST_VIDEO_ENCODER (self));
s = gst_caps_get_structure (output_state->caps, 0);
if (!strcmp (gst_structure_get_name (s), "video/x-h264") ||
!strcmp (gst_structure_get_name (s), "video/x-h265")) {
self->codec_data_in_bytestream = TRUE;
} else {
self->codec_data_in_bytestream = FALSE;
}
gst_video_codec_state_unref (output_state);
return TRUE;
}
/* The weird handling of cropping, alignment and everything is taken from
* platform/frameworks/media/libstagefright/colorconversion/ColorConversion.cpp
*/
static gboolean
gst_amc_video_enc_fill_buffer (GstAmcVideoEnc * self, GstBuffer * inbuf,
GstAmcBuffer * outbuf, const GstAmcBufferInfo * buffer_info)
{
GstVideoCodecState *input_state = self->input_state;
/* The fill_buffer runs in the same thread as set_format?
* then we can use state->info safely */
GstVideoInfo *info = &input_state->info;
if (buffer_info->size < self->color_format_info.frame_size)
return FALSE;
return gst_amc_color_format_copy (&self->color_format_info, outbuf,
buffer_info, info, inbuf, COLOR_FORMAT_COPY_IN);
}
static GstFlowReturn
gst_amc_video_enc_handle_output_frame (GstAmcVideoEnc * self,
GstAmcBuffer * buf, const GstAmcBufferInfo * buffer_info,
GstVideoCodecFrame * frame)
{
GstFlowReturn flow_ret = GST_FLOW_OK;
GstVideoEncoder *encoder = GST_VIDEO_ENCODER_CAST (self);
if (buffer_info->size > 0) {
GstBuffer *out_buf;
GstPad *srcpad;
if (buffer_info->flags & BUFFER_FLAG_PARTIAL_FRAME) {
GST_FIXME_OBJECT (self, "partial frames are currently not handled");
}
srcpad = GST_VIDEO_ENCODER_SRC_PAD (encoder);
out_buf =
gst_video_encoder_allocate_output_buffer (encoder, buffer_info->size);
gst_buffer_fill (out_buf, 0, buf->data + buffer_info->offset,
buffer_info->size);
GST_BUFFER_PTS (out_buf) =
gst_util_uint64_scale (buffer_info->presentation_time_us, GST_USECOND,
1);
if (frame) {
frame->output_buffer = out_buf;
flow_ret = gst_video_encoder_finish_frame (encoder, frame);
} else {
/* This sometimes happens at EOS or if the input is not properly framed,
* let's handle it gracefully by allocating a new buffer for the current
* caps and filling it
*/
GST_ERROR_OBJECT (self, "No corresponding frame found: buffer pts: %"
GST_TIME_FORMAT " presentation_time_us %" G_GUINT64_FORMAT,
GST_TIME_ARGS (GST_BUFFER_PTS (out_buf)),
(guint64) buffer_info->presentation_time_us);
flow_ret = gst_pad_push (srcpad, out_buf);
}
} else if (frame) {
flow_ret = gst_video_encoder_finish_frame (encoder, frame);
}
return flow_ret;
}
static void
gst_amc_video_enc_loop (GstAmcVideoEnc * self)
{
GstVideoEncoder *encoder = GST_VIDEO_ENCODER_CAST (self);
GstVideoCodecFrame *frame;
GstFlowReturn flow_ret = GST_FLOW_OK;
gboolean is_eos, is_codec_data;
GstAmcBufferInfo buffer_info;
GstAmcBuffer *buf;
gint idx;
GError *err = NULL;
GST_VIDEO_ENCODER_STREAM_LOCK (self);
retry:
GST_DEBUG_OBJECT (self, "Waiting for available output buffer");
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
/* Wait at most 100ms here, some codecs don't fail dequeueing if
* the codec is flushing, causing deadlocks during shutdown */
idx =
gst_amc_codec_dequeue_output_buffer (self->codec, &buffer_info, 100000,
&err);
GST_VIDEO_ENCODER_STREAM_LOCK (self);
/*} */
if (idx < 0 || self->amc_format) {
if (self->flushing) {
g_clear_error (&err);
goto flushing;
}
/* The comments from https://android.googlesource.com/platform/cts/+/android-4.3_r3.1/tests/tests/media/src/android/media/cts/EncodeDecodeTest.java
* line 539 says INFO_OUTPUT_FORMAT_CHANGED is not expected for an encoder
*/
if (self->amc_format || idx == INFO_OUTPUT_FORMAT_CHANGED) {
GstAmcFormat *format;
gchar *format_string;
GST_DEBUG_OBJECT (self, "Output format has changed");
format = (idx == INFO_OUTPUT_FORMAT_CHANGED) ?
gst_amc_codec_get_output_format (self->codec,
&err) : self->amc_format;
if (err) {
format = self->amc_format;
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
}
if (self->amc_format) {
if (format != self->amc_format)
gst_amc_format_free (self->amc_format);
self->amc_format = NULL;
}
if (!format)
goto format_error;
format_string = gst_amc_format_to_string (format, &err);
if (err) {
gst_amc_format_free (format);
goto format_error;
}
GST_DEBUG_OBJECT (self, "Got new output format: %s", format_string);
g_free (format_string);
if (!gst_amc_video_enc_set_src_caps (self, format)) {
gst_amc_format_free (format);
goto format_error;
}
gst_amc_format_free (format);
if (idx >= 0)
goto process_buffer;
goto retry;
}
switch (idx) {
case INFO_OUTPUT_BUFFERS_CHANGED:
/* Handled internally */
g_assert_not_reached ();
break;
case INFO_TRY_AGAIN_LATER:
GST_DEBUG_OBJECT (self, "Dequeueing output buffer timed out");
goto retry;
break;
case G_MININT:
GST_ERROR_OBJECT (self, "Failure dequeueing input buffer");
goto dequeue_error;
break;
default:
g_assert_not_reached ();
break;
}
goto retry;
}
process_buffer:
GST_DEBUG_OBJECT (self,
"Got output buffer at index %d: size %d time %" G_GINT64_FORMAT
" flags 0x%08x", idx, buffer_info.size, buffer_info.presentation_time_us,
buffer_info.flags);
buf = gst_amc_codec_get_output_buffer (self->codec, idx, &err);
if (err) {
if (self->flushing) {
g_clear_error (&err);
goto flushing;
}
goto failed_to_get_output_buffer;
} else if (!buf) {
goto got_null_output_buffer;
}
is_codec_data = FALSE;
/* The BUFFER_FLAG_CODEC_CONFIG logic is borrowed from
* gst-omx. see *_handle_output_frame in
* gstomxvideoenc.c and gstomxh264enc.c */
if ((buffer_info.flags & BUFFER_FLAG_CODEC_CONFIG)
&& buffer_info.size > 0) {
if (self->codec_data_in_bytestream) {
if (buffer_info.size > 4 &&
GST_READ_UINT32_BE (buf->data + buffer_info.offset) == 0x00000001) {
GList *l = NULL;
GstBuffer *hdrs;
GST_DEBUG_OBJECT (self, "got codecconfig in byte-stream format");
hdrs = gst_buffer_new_and_alloc (buffer_info.size);
gst_buffer_fill (hdrs, 0, buf->data + buffer_info.offset,
buffer_info.size);
GST_BUFFER_PTS (hdrs) =
gst_util_uint64_scale (buffer_info.presentation_time_us,
GST_USECOND, 1);
l = g_list_append (l, hdrs);
gst_video_encoder_set_headers (encoder, l);
is_codec_data = TRUE;
}
} else {
GstBuffer *codec_data;
GstVideoCodecState *output_state =
gst_video_encoder_get_output_state (GST_VIDEO_ENCODER (self));
GST_DEBUG_OBJECT (self, "Handling codec data");
codec_data = gst_buffer_new_and_alloc (buffer_info.size);
gst_buffer_fill (codec_data, 0, buf->data + buffer_info.offset,
buffer_info.size);
output_state->codec_data = codec_data;
gst_video_codec_state_unref (output_state);
is_codec_data = TRUE;
if (!gst_video_encoder_negotiate (encoder))
flow_ret = GST_FLOW_NOT_NEGOTIATED;
}
}
is_eos = !!(buffer_info.flags & BUFFER_FLAG_END_OF_STREAM);
if (flow_ret == GST_FLOW_OK && !is_codec_data) {
frame =
_find_nearest_frame (self,
gst_util_uint64_scale (buffer_info.presentation_time_us, GST_USECOND,
1));
flow_ret =
gst_amc_video_enc_handle_output_frame (self, buf, &buffer_info, frame);
}
gst_amc_buffer_free (buf);
buf = NULL;
if (!gst_amc_codec_release_output_buffer (self->codec, idx, FALSE, &err)) {
if (self->flushing) {
g_clear_error (&err);
goto flushing;
}
goto failed_release;
}
if (is_eos || flow_ret == GST_FLOW_EOS) {
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
g_mutex_lock (&self->drain_lock);
if (self->draining) {
GST_DEBUG_OBJECT (self, "Drained");
self->draining = FALSE;
g_cond_broadcast (&self->drain_cond);
} else if (flow_ret == GST_FLOW_OK) {
GST_DEBUG_OBJECT (self, "Component signalled EOS");
flow_ret = GST_FLOW_EOS;
}
g_mutex_unlock (&self->drain_lock);
GST_VIDEO_ENCODER_STREAM_LOCK (self);
} else {
GST_DEBUG_OBJECT (self, "Finished frame: %s", gst_flow_get_name (flow_ret));
}
self->downstream_flow_ret = flow_ret;
if (flow_ret != GST_FLOW_OK)
goto flow_error;
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
return;
dequeue_error:
{
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
gst_pad_push_event (GST_VIDEO_ENCODER_SRC_PAD (self), gst_event_new_eos ());
gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self));
self->downstream_flow_ret = GST_FLOW_ERROR;
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
g_mutex_lock (&self->drain_lock);
self->draining = FALSE;
g_cond_broadcast (&self->drain_cond);
g_mutex_unlock (&self->drain_lock);
return;
}
format_error:
{
if (err)
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
else
GST_ELEMENT_ERROR (self, LIBRARY, FAILED, (NULL),
("Failed to handle format"));
gst_pad_push_event (GST_VIDEO_ENCODER_SRC_PAD (self), gst_event_new_eos ());
gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self));
self->downstream_flow_ret = GST_FLOW_ERROR;
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
g_mutex_lock (&self->drain_lock);
self->draining = FALSE;
g_cond_broadcast (&self->drain_cond);
g_mutex_unlock (&self->drain_lock);
return;
}
failed_release:
{
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
gst_pad_push_event (GST_VIDEO_ENCODER_SRC_PAD (self), gst_event_new_eos ());
gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self));
self->downstream_flow_ret = GST_FLOW_ERROR;
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
g_mutex_lock (&self->drain_lock);
self->draining = FALSE;
g_cond_broadcast (&self->drain_cond);
g_mutex_unlock (&self->drain_lock);
return;
}
flushing:
{
GST_DEBUG_OBJECT (self, "Flushing -- stopping task");
gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self));
self->downstream_flow_ret = GST_FLOW_FLUSHING;
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
return;
}
flow_error:
{
if (flow_ret == GST_FLOW_EOS) {
GST_DEBUG_OBJECT (self, "EOS");
gst_pad_push_event (GST_VIDEO_ENCODER_SRC_PAD (self),
gst_event_new_eos ());
gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self));
} else if (flow_ret == GST_FLOW_NOT_LINKED || flow_ret < GST_FLOW_EOS) {
GST_ELEMENT_FLOW_ERROR (self, flow_ret);
gst_pad_push_event (GST_VIDEO_ENCODER_SRC_PAD (self),
gst_event_new_eos ());
gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self));
}
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
g_mutex_lock (&self->drain_lock);
self->draining = FALSE;
g_cond_broadcast (&self->drain_cond);
g_mutex_unlock (&self->drain_lock);
return;
}
failed_to_get_output_buffer:
{
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
gst_pad_push_event (GST_VIDEO_ENCODER_SRC_PAD (self), gst_event_new_eos ());
gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self));
self->downstream_flow_ret = GST_FLOW_ERROR;
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
g_mutex_lock (&self->drain_lock);
self->draining = FALSE;
g_cond_broadcast (&self->drain_cond);
g_mutex_unlock (&self->drain_lock);
return;
}
got_null_output_buffer:
{
GST_ELEMENT_ERROR (self, LIBRARY, SETTINGS, (NULL),
("Got no output buffer"));
gst_pad_push_event (GST_VIDEO_ENCODER_SRC_PAD (self), gst_event_new_eos ());
gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self));
self->downstream_flow_ret = GST_FLOW_ERROR;
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
g_mutex_lock (&self->drain_lock);
self->draining = FALSE;
g_cond_broadcast (&self->drain_cond);
g_mutex_unlock (&self->drain_lock);
return;
}
}
static gboolean
gst_amc_video_enc_start (GstVideoEncoder * encoder)
{
GstAmcVideoEnc *self;
self = GST_AMC_VIDEO_ENC (encoder);
self->last_upstream_ts = 0;
self->drained = TRUE;
self->downstream_flow_ret = GST_FLOW_OK;
self->started = FALSE;
self->flushing = TRUE;
return TRUE;
}
static gboolean
gst_amc_video_enc_stop (GstVideoEncoder * encoder)
{
GstAmcVideoEnc *self;
GError *err = NULL;
self = GST_AMC_VIDEO_ENC (encoder);
GST_DEBUG_OBJECT (self, "Stopping encoder");
self->flushing = TRUE;
if (self->started) {
gst_amc_codec_flush (self->codec, &err);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
gst_amc_codec_stop (self->codec, &err);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
self->started = FALSE;
}
gst_pad_stop_task (GST_VIDEO_ENCODER_SRC_PAD (encoder));
self->downstream_flow_ret = GST_FLOW_FLUSHING;
self->drained = TRUE;
g_mutex_lock (&self->drain_lock);
self->draining = FALSE;
g_cond_broadcast (&self->drain_cond);
g_mutex_unlock (&self->drain_lock);
if (self->input_state)
gst_video_codec_state_unref (self->input_state);
self->input_state = NULL;
if (self->amc_format) {
gst_amc_format_free (self->amc_format);
self->amc_format = NULL;
}
GST_DEBUG_OBJECT (self, "Stopped encoder");
return TRUE;
}
static gboolean
gst_amc_video_enc_set_format (GstVideoEncoder * encoder,
GstVideoCodecState * state)
{
GstAmcVideoEnc *self;
GstAmcFormat *format = NULL;
GstCaps *allowed_caps = NULL;
gboolean is_format_change = FALSE;
gboolean needs_disable = FALSE;
gchar *format_string;
gboolean r = FALSE;
GError *err = NULL;
self = GST_AMC_VIDEO_ENC (encoder);
GST_DEBUG_OBJECT (self, "Setting new caps %" GST_PTR_FORMAT, state->caps);
/* Check if the caps change is a real format change or if only irrelevant
* parts of the caps have changed or nothing at all.
*/
is_format_change |= self->color_format_info.width != state->info.width;
is_format_change |= self->color_format_info.height != state->info.height;
needs_disable = self->started;
/* If the component is not started and a real format change happens
* we have to restart the component. If no real format change
* happened we can just exit here.
*/
if (needs_disable && !is_format_change) {
/* Framerate or something minor changed */
if (self->input_state)
gst_video_codec_state_unref (self->input_state);
self->input_state = gst_video_codec_state_ref (state);
GST_DEBUG_OBJECT (self,
"Already running and caps did not change the format");
return TRUE;
}
if (needs_disable && is_format_change) {
gst_amc_video_enc_drain (self);
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
gst_amc_video_enc_stop (GST_VIDEO_ENCODER (self));
GST_VIDEO_ENCODER_STREAM_LOCK (self);
gst_amc_video_enc_close (GST_VIDEO_ENCODER (self));
if (!gst_amc_video_enc_open (GST_VIDEO_ENCODER (self))) {
GST_ERROR_OBJECT (self, "Failed to open codec again");
return FALSE;
}
if (!gst_amc_video_enc_start (GST_VIDEO_ENCODER (self))) {
GST_ERROR_OBJECT (self, "Failed to start codec again");
}
}
/* srcpad task is not running at this point */
if (self->input_state)
gst_video_codec_state_unref (self->input_state);
self->input_state = NULL;
GST_DEBUG_OBJECT (self, "picking an output format ...");
allowed_caps = gst_pad_get_allowed_caps (GST_VIDEO_ENCODER_SRC_PAD (encoder));
if (!allowed_caps) {
GST_DEBUG_OBJECT (self, "... but no peer, using template caps");
allowed_caps =
gst_pad_get_pad_template_caps (GST_VIDEO_ENCODER_SRC_PAD (encoder));
}
GST_DEBUG_OBJECT (self, "chose caps %" GST_PTR_FORMAT, allowed_caps);
allowed_caps = gst_caps_truncate (allowed_caps);
format = create_amc_format (self, state, allowed_caps);
if (!format)
goto quit;
format_string = gst_amc_format_to_string (format, &err);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
GST_DEBUG_OBJECT (self, "Configuring codec with format: %s",
GST_STR_NULL (format_string));
g_free (format_string);
if (!gst_amc_codec_configure (self->codec, format, NULL, &err)) {
GST_ERROR_OBJECT (self, "Failed to configure codec");
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
goto quit;
}
if (!gst_amc_codec_start (self->codec, &err)) {
GST_ERROR_OBJECT (self, "Failed to start codec");
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
goto quit;
}
self->amc_format = format;
format = NULL;
self->input_state = gst_video_codec_state_ref (state);
self->started = TRUE;
/* Start the srcpad loop again */
self->flushing = FALSE;
self->downstream_flow_ret = GST_FLOW_OK;
gst_pad_start_task (GST_VIDEO_ENCODER_SRC_PAD (self),
(GstTaskFunction) gst_amc_video_enc_loop, encoder, NULL);
r = TRUE;
quit:
if (allowed_caps)
gst_caps_unref (allowed_caps);
if (format)
gst_amc_format_free (format);
return r;
}
static gboolean
gst_amc_video_enc_flush (GstVideoEncoder * encoder)
{
GstAmcVideoEnc *self;
GError *err = NULL;
self = GST_AMC_VIDEO_ENC (encoder);
GST_DEBUG_OBJECT (self, "Flushing encoder");
if (!self->started) {
GST_DEBUG_OBJECT (self, "Codec not started yet");
return TRUE;
}
self->flushing = TRUE;
gst_amc_codec_flush (self->codec, &err);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
/* Wait until the srcpad loop is finished,
* unlock GST_VIDEO_ENCODER_STREAM_LOCK to prevent deadlocks
* caused by using this lock from inside the loop function */
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
GST_PAD_STREAM_LOCK (GST_VIDEO_ENCODER_SRC_PAD (self));
GST_PAD_STREAM_UNLOCK (GST_VIDEO_ENCODER_SRC_PAD (self));
GST_VIDEO_ENCODER_STREAM_LOCK (self);
self->flushing = FALSE;
/* Start the srcpad loop again */
self->last_upstream_ts = 0;
self->drained = TRUE;
self->downstream_flow_ret = GST_FLOW_OK;
gst_pad_start_task (GST_VIDEO_ENCODER_SRC_PAD (self),
(GstTaskFunction) gst_amc_video_enc_loop, encoder, NULL);
GST_DEBUG_OBJECT (self, "Flush encoder");
return TRUE;
}
static GstFlowReturn
gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder,
GstVideoCodecFrame * frame)
{
GstAmcVideoEnc *self;
gint idx;
GstAmcBuffer *buf;
GstAmcBufferInfo buffer_info;
GstClockTime timestamp, duration, timestamp_offset = 0;
BufferIdentification *id;
GError *err = NULL;
self = GST_AMC_VIDEO_ENC (encoder);
GST_DEBUG_OBJECT (self, "Handling frame");
if (!self->started) {
GST_ERROR_OBJECT (self, "Codec not started yet");
gst_video_codec_frame_unref (frame);
return GST_FLOW_NOT_NEGOTIATED;
}
if (self->flushing)
goto flushing;
if (self->downstream_flow_ret != GST_FLOW_OK)
goto downstream_error;
timestamp = frame->pts;
duration = frame->duration;
if (GST_VIDEO_CODEC_FRAME_IS_FORCE_KEYFRAME (frame)) {
if (gst_amc_codec_request_key_frame (self->codec, &err)) {
GST_DEBUG_OBJECT (self, "Passed keyframe request to MediaCodec");
}
if (err) {
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
g_clear_error (&err);
}
}
again:
/* Make sure to release the base class stream lock, otherwise
* _loop() can't call _finish_frame() and we might block forever
* because no input buffers are released */
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
/* Wait at most 100ms here, some codecs don't fail dequeueing if
* the codec is flushing, causing deadlocks during shutdown */
idx = gst_amc_codec_dequeue_input_buffer (self->codec, 100000, &err);
GST_VIDEO_ENCODER_STREAM_LOCK (self);
if (idx < 0) {
if (self->flushing || self->downstream_flow_ret == GST_FLOW_FLUSHING) {
g_clear_error (&err);
goto flushing;
}
switch (idx) {
case INFO_TRY_AGAIN_LATER:
GST_DEBUG_OBJECT (self, "Dequeueing input buffer timed out");
goto again; /* next try */
break;
case G_MININT:
GST_ERROR_OBJECT (self, "Failed to dequeue input buffer");
goto dequeue_error;
default:
g_assert_not_reached ();
break;
}
goto again;
}
if (self->flushing) {
memset (&buffer_info, 0, sizeof (buffer_info));
gst_amc_codec_queue_input_buffer (self->codec, idx, &buffer_info, NULL);
goto flushing;
}
if (self->downstream_flow_ret != GST_FLOW_OK) {
memset (&buffer_info, 0, sizeof (buffer_info));
gst_amc_codec_queue_input_buffer (self->codec, idx, &buffer_info, &err);
if (err && !self->flushing)
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
g_clear_error (&err);
goto downstream_error;
}
/* Now handle the frame */
/* Copy the buffer content in chunks of size as requested
* by the port */
buf = gst_amc_codec_get_input_buffer (self->codec, idx, &err);
if (err)
goto failed_to_get_input_buffer;
else if (!buf)
goto got_null_input_buffer;
memset (&buffer_info, 0, sizeof (buffer_info));
buffer_info.offset = 0;
buffer_info.size = MIN (self->color_format_info.frame_size, buf->size);
gst_amc_buffer_set_position_and_limit (buf, NULL, buffer_info.offset,
buffer_info.size);
if (!gst_amc_video_enc_fill_buffer (self, frame->input_buffer, buf,
&buffer_info)) {
memset (&buffer_info, 0, sizeof (buffer_info));
gst_amc_codec_queue_input_buffer (self->codec, idx, &buffer_info, &err);
if (err && !self->flushing)
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
g_clear_error (&err);
gst_amc_buffer_free (buf);
buf = NULL;
goto buffer_fill_error;
}
gst_amc_buffer_free (buf);
buf = NULL;
if (timestamp != GST_CLOCK_TIME_NONE) {
buffer_info.presentation_time_us =
gst_util_uint64_scale (timestamp + timestamp_offset, 1, GST_USECOND);
self->last_upstream_ts = timestamp + timestamp_offset;
}
if (duration != GST_CLOCK_TIME_NONE)
self->last_upstream_ts += duration;
id = buffer_identification_new (timestamp + timestamp_offset);
if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame))
buffer_info.flags |= BUFFER_FLAG_SYNC_FRAME;
gst_video_codec_frame_set_user_data (frame, id,
(GDestroyNotify) buffer_identification_free);
GST_DEBUG_OBJECT (self,
"Queueing buffer %d: size %d time %" G_GINT64_FORMAT " flags 0x%08x",
idx, buffer_info.size, buffer_info.presentation_time_us,
buffer_info.flags);
if (!gst_amc_codec_queue_input_buffer (self->codec, idx, &buffer_info, &err)) {
if (self->flushing) {
g_clear_error (&err);
goto flushing;
}
goto queue_error;
}
self->drained = FALSE;
gst_video_codec_frame_unref (frame);
return self->downstream_flow_ret;
downstream_error:
{
GST_ERROR_OBJECT (self, "Downstream returned %s",
gst_flow_get_name (self->downstream_flow_ret));
gst_video_codec_frame_unref (frame);
return self->downstream_flow_ret;
}
failed_to_get_input_buffer:
{
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
gst_video_codec_frame_unref (frame);
return GST_FLOW_ERROR;
}
got_null_input_buffer:
{
GST_ELEMENT_ERROR (self, LIBRARY, SETTINGS, (NULL),
("Got no input buffer"));
gst_video_codec_frame_unref (frame);
return GST_FLOW_ERROR;
}
buffer_fill_error:
{
GST_ELEMENT_ERROR (self, RESOURCE, WRITE, (NULL),
("Failed to write input into the amc buffer(write %dB to a %"
G_GSIZE_FORMAT "B buffer)", self->color_format_info.frame_size,
buf->size));
gst_video_codec_frame_unref (frame);
return GST_FLOW_ERROR;
}
dequeue_error:
{
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
gst_video_codec_frame_unref (frame);
return GST_FLOW_ERROR;
}
queue_error:
{
GST_ELEMENT_ERROR_FROM_ERROR (self, err);
gst_video_codec_frame_unref (frame);
return GST_FLOW_ERROR;
}
flushing:
{
GST_DEBUG_OBJECT (self, "Flushing -- returning FLUSHING");
gst_video_codec_frame_unref (frame);
return GST_FLOW_FLUSHING;
}
}
static GstFlowReturn
gst_amc_video_enc_finish (GstVideoEncoder * encoder)
{
GstAmcVideoEnc *self;
self = GST_AMC_VIDEO_ENC (encoder);
return gst_amc_video_enc_drain (self);
}
static GstFlowReturn
gst_amc_video_enc_drain (GstAmcVideoEnc * self)
{
GstFlowReturn ret;
gint idx;
GError *err = NULL;
GST_DEBUG_OBJECT (self, "Draining codec");
if (!self->started) {
GST_DEBUG_OBJECT (self, "Codec not started yet");
return GST_FLOW_OK;
}
/* Don't send drain buffer twice, this doesn't work */
if (self->drained) {
GST_DEBUG_OBJECT (self, "Codec is drained already");
return GST_FLOW_OK;
}
/* Make sure to release the base class stream lock, otherwise
* _loop() can't call _finish_frame() and we might block forever
* because no input buffers are released */
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
/* Send an EOS buffer to the component and let the base
* class drop the EOS event. We will send it later when
* the EOS buffer arrives on the output port.
* Wait at most 0.5s here. */
idx = gst_amc_codec_dequeue_input_buffer (self->codec, 500000, &err);
GST_VIDEO_ENCODER_STREAM_LOCK (self);
if (idx >= 0) {
GstAmcBuffer *buf;
GstAmcBufferInfo buffer_info;
buf = gst_amc_codec_get_input_buffer (self->codec, idx, &err);
if (buf) {
GST_VIDEO_ENCODER_STREAM_UNLOCK (self);
g_mutex_lock (&self->drain_lock);
self->draining = TRUE;
memset (&buffer_info, 0, sizeof (buffer_info));
buffer_info.size = 0;
buffer_info.presentation_time_us =
gst_util_uint64_scale (self->last_upstream_ts, 1, GST_USECOND);
buffer_info.flags |= BUFFER_FLAG_END_OF_STREAM;
gst_amc_buffer_set_position_and_limit (buf, NULL, 0, 0);
gst_amc_buffer_free (buf);
buf = NULL;
if (gst_amc_codec_queue_input_buffer (self->codec, idx, &buffer_info,
&err)) {
GST_DEBUG_OBJECT (self, "Waiting until codec is drained");
g_cond_wait (&self->drain_cond, &self->drain_lock);
GST_DEBUG_OBJECT (self, "Drained codec");
ret = GST_FLOW_OK;
} else {
GST_ERROR_OBJECT (self, "Failed to queue input buffer");
if (self->flushing) {
g_clear_error (&err);
ret = GST_FLOW_FLUSHING;
} else {
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
ret = GST_FLOW_ERROR;
}
}
self->drained = TRUE;
self->draining = FALSE;
g_mutex_unlock (&self->drain_lock);
GST_VIDEO_ENCODER_STREAM_LOCK (self);
} else {
GST_ERROR_OBJECT (self, "Failed to get buffer for EOS: %d", idx);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
ret = GST_FLOW_ERROR;
}
} else {
GST_ERROR_OBJECT (self, "Failed to acquire buffer for EOS: %d", idx);
if (err)
GST_ELEMENT_WARNING_FROM_ERROR (self, err);
ret = GST_FLOW_ERROR;
}
return ret;
}
Reference in a new issue View git blame Copy permalink