mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-12-19 06:46:38 +00:00
Code Issues 1 Projects Releases Wiki Activity

applemedia: Add HEVC support to vtenc and vtdec

Browse source 
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2754>
This commit is contained in:
Piotr Brzeziński 2022-07-13 10:30:49 +02:00 committed by GStreamer Marge Bot
parent 1dd29a2564
commit 6bf15124e7
2 changed files with 129 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
subprojects/gst-plugins-bad/sys/applemedia/vtdec.c
View file

@ -105,6 +105,8 @@ static void gst_vtdec_session_output_callback (void
CMTime duration);
static gboolean compute_h264_decode_picture_buffer_length (GstVtdec * vtdec,
GstBuffer * codec_data, int *length);
static gboolean compute_hevc_decode_picture_buffer_length (GstVtdec * vtdec,
GstBuffer * codec_data, int *length);
static gboolean gst_vtdec_compute_reorder_queue_length (GstVtdec * vtdec,
CMVideoCodecType cm_format, GstBuffer * codec_data);
static void gst_vtdec_set_latency (GstVtdec * vtdec);
@ -115,6 +117,8 @@ static GstStaticPadTemplate gst_vtdec_sink_template =
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS ("video/x-h264, stream-format=avc, alignment=au,"
" width=(int)[1, MAX], height=(int)[1, MAX];"
"video/x-h265, stream-format=(string){ hev1, hvc1 }, alignment=au,"
" width=(int)[1, MAX], height=(int)[1, MAX];"
"video/mpeg, mpegversion=2, systemstream=false, parsed=true;"
"image/jpeg;"
@ -535,6 +539,8 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
caps_name = gst_structure_get_name (structure);
if (!strcmp (caps_name, "video/x-h264")) {
cm_format = kCMVideoCodecType_H264;
} else if (!strcmp (caps_name, "video/x-h265")) {
cm_format = kCMVideoCodecType_HEVC;
} else if (!strcmp (caps_name, "video/mpeg")) {
cm_format = kCMVideoCodecType_MPEG2Video;
} else if (!strcmp (caps_name, "image/jpeg")) {
@ -551,7 +557,9 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state)
}
}
if (cm_format == kCMVideoCodecType_H264 && state->codec_data == NULL) {
if ((cm_format == kCMVideoCodecType_H264
|| cm_format == kCMVideoCodecType_HEVC)
&& state->codec_data == NULL) {
GST_INFO_OBJECT (vtdec, "no codec data, wait for one");
return TRUE;
}
@ -788,7 +796,12 @@ create_format_description_from_codec_data (GstVtdec * vtdec,
gst_buffer_map (codec_data, &map, GST_MAP_READ);
atoms = CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
&kCFTypeDictionaryValueCallBacks);
gst_vtutil_dict_set_data (atoms, CFSTR ("avcC"), map.data, map.size);
if (cm_format == kCMVideoCodecType_HEVC)
gst_vtutil_dict_set_data (atoms, CFSTR ("hvcC"), map.data, map.size);
else
gst_vtutil_dict_set_data (atoms, CFSTR ("avcC"), map.data, map.size);
gst_vtutil_dict_set_object (extensions,
CFSTR ("SampleDescriptionExtensionAtoms"), (CFTypeRef *) atoms);
gst_buffer_unmap (codec_data, &map);
@ -1148,6 +1161,11 @@ gst_vtdec_compute_reorder_queue_length (GstVtdec * vtdec,
&vtdec->reorder_queue_length)) {
return FALSE;
}
} else if (cm_format == kCMVideoCodecType_HEVC) {
if (!compute_hevc_decode_picture_buffer_length (vtdec, codec_data,
&vtdec->reorder_queue_length)) {
return FALSE;
}
} else {
vtdec->reorder_queue_length = 0;
}
@ -1195,6 +1213,34 @@ compute_h264_decode_picture_buffer_length (GstVtdec * vtdec,
return TRUE;
}
static gboolean
compute_hevc_decode_picture_buffer_length (GstVtdec * vtdec,
GstBuffer * codec_data, int *length)
{
/* This value should be level dependent (table A.8)
* but let's assume the maximum possible one for simplicity. */
const gint max_luma_ps = 35651584;
const gint max_dpb_pic_buf = 6;
gint max_dbp_size, pic_size_samples_y;
if (vtdec->video_info.width == 0 || vtdec->video_info.height == 0)
return FALSE;
/* A.4.2 */
pic_size_samples_y = vtdec->video_info.width * vtdec->video_info.height;
if (pic_size_samples_y <= (max_luma_ps >> 2))
max_dbp_size = max_dpb_pic_buf * 4;
else if (pic_size_samples_y <= (max_luma_ps >> 1))
max_dbp_size = max_dpb_pic_buf * 2;
else if (pic_size_samples_y <= ((3 * max_luma_ps) >> 2))
max_dbp_size = (max_dpb_pic_buf * 4) / 3;
else
max_dbp_size = max_dpb_pic_buf;
*length = MIN (max_dbp_size, 16);
return TRUE;
}
static void
gst_vtdec_set_latency (GstVtdec * vtdec)
{

100
subprojects/gst-plugins-bad/sys/applemedia/vtenc.c
View file

@ -71,7 +71,6 @@
#include "vtutil.h"
#include <gst/pbutils/codec-utils.h>
#define VTENC_DEFAULT_USAGE 6 /* Profile: Baseline Level: 2.1 */
#define VTENC_DEFAULT_BITRATE 0
#define VTENC_DEFAULT_FRAME_REORDERING TRUE
#define VTENC_DEFAULT_REALTIME FALSE
@ -239,7 +238,6 @@ gst_vtenc_base_init (GstVTEncClass * klass)
gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, caps));
}
src_caps = gst_caps_new_simple (codec_details->mimetype,
"width", GST_TYPE_INT_RANGE, min_width, max_width,
"height", GST_TYPE_INT_RANGE, min_height, max_height,
@ -268,6 +266,11 @@ gst_vtenc_base_init (GstVTEncClass * klass)
"stream-format", G_TYPE_STRING, "avc",
"alignment", G_TYPE_STRING, "au", NULL);
break;
case kCMVideoCodecType_HEVC:
gst_structure_set (gst_caps_get_structure (src_caps, 0),
"stream-format", G_TYPE_STRING, "hvc1",
"alignment", G_TYPE_STRING, "au", NULL);
break;
case GST_kCMVideoCodecType_Some_AppleProRes:
if (g_strcmp0 (codec_details->mimetype, "video/x-prores") == 0) {
G_GNUC_BEGIN_IGNORE_DEPRECATIONS;
@ -720,7 +723,7 @@ gst_vtenc_stop (GstVideoEncoder * enc)
}
static CFStringRef
gst_vtenc_profile_level_key (GstVTEnc * self, const gchar * profile,
gst_vtenc_h264_profile_level_key (GstVTEnc * self, const gchar * profile,
const gchar * level_arg)
{
char level[64];
@ -763,6 +766,37 @@ gst_vtenc_profile_level_key (GstVTEnc * self, const gchar * profile,
return ret;
}
static CFStringRef
gst_vtenc_hevc_profile_level_key (GstVTEnc * self, const gchar * profile,
const gchar * level_arg)
{
gchar *key = NULL;
CFStringRef ret = NULL;
if (profile == NULL || !strcmp (profile, "main"))
profile = "Main";
else if (!strcmp (profile, "main-10"))
profile = "Main10";
else if (!strcmp (profile, "main-422-10"))
/* TODO: this should probably be guarded with a version check (macOS 12.3+ / iOS 15.4+)
* https://developer.apple.com/documentation/videotoolbox/kvtprofilelevel_hevc_main10_autolevel */
profile = "Main42210";
else {
GST_ERROR_OBJECT (self, "invalid profile: %s", profile);
return ret;
}
/* VT does not support specific levels for HEVC */
key = g_strdup_printf ("HEVC_%s_AutoLevel", profile);
ret = CFStringCreateWithBytes (NULL, (const guint8 *) key, strlen (key),
kCFStringEncodingASCII, 0);
GST_INFO_OBJECT (self, "negotiated profile and level %s", key);
g_free (key);
return ret;
}
static gboolean
gst_vtenc_negotiate_profile_and_level (GstVTEnc * self, GstStructure * s)
{
@ -771,9 +805,16 @@ gst_vtenc_negotiate_profile_and_level (GstVTEnc * self, GstStructure * s)
if (self->profile_level)
CFRelease (self->profile_level);
self->profile_level = gst_vtenc_profile_level_key (self, profile, level);
if (self->specific_format_id == kCMVideoCodecType_HEVC)
self->profile_level =
gst_vtenc_hevc_profile_level_key (self, profile, level);
else
self->profile_level =
gst_vtenc_h264_profile_level_key (self, profile, level);
if (self->profile_level == NULL) {
GST_ERROR_OBJECT (self, "unsupported h264 profile '%s' or level '%s'",
GST_ERROR_OBJECT (self, "unsupported profile '%s' or level '%s'",
profile, level);
return FALSE;
}
@ -822,6 +863,11 @@ gst_vtenc_negotiate_specific_format_details (GstVideoEncoder * enc)
if (!gst_vtenc_negotiate_profile_and_level (self, s))
goto fail;
break;
case kCMVideoCodecType_HEVC:
self->specific_format_id = kCMVideoCodecType_HEVC;
if (!gst_vtenc_negotiate_profile_and_level (self, s))
goto fail;
break;
case GST_kCMVideoCodecType_Some_AppleProRes:
if (g_strcmp0 (self->details->mimetype, "video/x-prores") != 0) {
GST_ERROR_OBJECT (self, "format_id == %i mimetype must be Apple "
@ -931,35 +977,48 @@ gst_vtenc_negotiate_downstream (GstVTEnc * self, CMSampleBufferRef sbuf)
switch (self->details->format_id) {
case kCMVideoCodecType_H264:
case kCMVideoCodecType_HEVC:
{
CMFormatDescriptionRef fmt;
CFDictionaryRef atoms;
CFStringRef avccKey;
CFDataRef avcc;
CFStringRef boxKey;
CFDataRef box;
guint8 *codec_data;
gsize codec_data_size;
GstBuffer *codec_data_buf;
guint8 sps[3];
guint8 sps[12];
fmt = CMSampleBufferGetFormatDescription (sbuf);
atoms = CMFormatDescriptionGetExtension (fmt,
kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms);
avccKey = CFStringCreateWithCString (NULL, "avcC", kCFStringEncodingUTF8);
avcc = CFDictionaryGetValue (atoms, avccKey);
CFRelease (avccKey);
codec_data_size = CFDataGetLength (avcc);
if (self->details->format_id == kCMVideoCodecType_HEVC)
boxKey =
CFStringCreateWithCString (NULL, "hvcC", kCFStringEncodingUTF8);
else
boxKey =
CFStringCreateWithCString (NULL, "avcC", kCFStringEncodingUTF8);
box = CFDictionaryGetValue (atoms, boxKey);
CFRelease (boxKey);
codec_data_size = CFDataGetLength (box);
codec_data = g_malloc (codec_data_size);
CFDataGetBytes (avcc, CFRangeMake (0, codec_data_size), codec_data);
CFDataGetBytes (box, CFRangeMake (0, codec_data_size), codec_data);
codec_data_buf = gst_buffer_new_wrapped (codec_data, codec_data_size);
gst_structure_set (s, "codec_data", GST_TYPE_BUFFER, codec_data_buf,
NULL);
sps[0] = codec_data[1];
sps[1] = codec_data[2] & ~0xDF;
sps[2] = codec_data[3];
gst_codec_utils_h264_caps_set_level_and_profile (caps, sps, 3);
if (self->details->format_id == kCMVideoCodecType_HEVC) {
sps[0] = codec_data[1];
sps[11] = codec_data[12];
gst_codec_utils_h265_caps_set_level_tier_and_profile (caps, sps, 12);
} else {
sps[0] = codec_data[1];
sps[1] = codec_data[2] & ~0xDF;
sps[2] = codec_data[3];
gst_codec_utils_h264_caps_set_level_and_profile (caps, sps, 3);
}
gst_buffer_unref (codec_data_buf);
}
@ -1189,7 +1248,7 @@ gst_vtenc_create_session (GstVTEnc * self)
(gdouble) self->negotiated_fps_n / (gdouble) self->negotiated_fps_d);
/*
* https://developer.apple.com/documentation/videotoolbox/vtcompressionsession/compression_properties/profile_and_level_constants
* https://developer.apple.com/documentation/videotoolbox/kvtcompressionpropertykey_profilelevel
*/
status = VTSessionSetProperty (session,
kVTCompressionPropertyKey_ProfileLevel, self->profile_level);
@ -1885,8 +1944,11 @@ gst_vtenc_register (GstPlugin * plugin,
static const GstVTEncoderDetails gst_vtenc_codecs[] = {
{"H.264", "h264", "video/x-h264", kCMVideoCodecType_H264, FALSE},
{"H.265/HEVC", "h265", "video/x-h265", kCMVideoCodecType_HEVC, FALSE},
#ifndef HAVE_IOS
{"H.264 (HW only)", "h264_hw", "video/x-h264", kCMVideoCodecType_H264, TRUE},
{"H.265/HEVC (HW only)", "h265_hw", "video/x-h265", kCMVideoCodecType_HEVC,
TRUE},
#endif
{"Apple ProRes", "prores", "video/x-prores",
GST_kCMVideoCodecType_Some_AppleProRes, FALSE},