vtenc: Add max-frame-delay property

This controls the number of frames allowed in the compression window.
Not all encoders and implementations support it; I've only managed to
successfully use it with ProRes.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7232>
This commit is contained in:
Nirbheek Chauhan 2024-07-27 06:27:14 +05:30 committed by GStreamer Marge Bot
parent 30d2b8895a
commit f8711239e4
2 changed files with 45 additions and 4 deletions

View file

@ -194,6 +194,7 @@ enum
PROP_PRESERVE_ALPHA, PROP_PRESERVE_ALPHA,
PROP_RATE_CONTROL, PROP_RATE_CONTROL,
PROP_DATA_RATE_LIMITS, PROP_DATA_RATE_LIMITS,
PROP_MAX_FRAME_DELAY,
}; };
typedef struct _GstVTEncFrame GstVTEncFrame; typedef struct _GstVTEncFrame GstVTEncFrame;
@ -235,6 +236,8 @@ static void gst_vtenc_session_configure_max_keyframe_interval (GstVTEnc * self,
VTCompressionSessionRef session, gint interval); VTCompressionSessionRef session, gint interval);
static void gst_vtenc_session_configure_max_keyframe_interval_duration static void gst_vtenc_session_configure_max_keyframe_interval_duration
(GstVTEnc * self, VTCompressionSessionRef session, gdouble duration); (GstVTEnc * self, VTCompressionSessionRef session, gdouble duration);
static void gst_vtenc_session_configure_max_frame_delay (GstVTEnc * self,
VTCompressionSessionRef session, int delay);
static void gst_vtenc_session_configure_bitrate (GstVTEnc * self, static void gst_vtenc_session_configure_bitrate (GstVTEnc * self,
VTCompressionSessionRef session, guint bitrate); VTCompressionSessionRef session, guint bitrate);
static OSStatus gst_vtenc_session_configure_property_int (GstVTEnc * self, static OSStatus gst_vtenc_session_configure_property_int (GstVTEnc * self,
@ -501,9 +504,19 @@ gst_vtenc_class_init (GstVTEncClass * klass)
/* /*
* H264 doesn't support alpha components, and H265 uses a separate element for encoding * H264 doesn't support alpha components, and H265 uses a separate element for encoding
* with alpha, so only add the property for prores * with alpha, so only add the preserve-alpha property for ProRes.
*
* MaxFrameDelayCount seems to only be supported with ProRes
*/ */
if (g_strcmp0 (G_OBJECT_CLASS_NAME (klass), "vtenc_prores") == 0) { if (g_strcmp0 (G_OBJECT_CLASS_NAME (klass), "vtenc_prores") == 0) {
/**
* Since: 1.26
*/
g_object_class_install_property (gobject_class, PROP_MAX_FRAME_DELAY,
g_param_spec_int ("max-frame-delay", "Maximum Frame Delay",
"Maximum frames allowed in the compression window (-1 = unlimited)",
-1, G_MAXINT, -1,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
/** /**
* vtenc_prores:preserve-alpha * vtenc_prores:preserve-alpha
* *
@ -638,6 +651,16 @@ gst_vtenc_set_max_keyframe_interval_duration (GstVTEnc * self,
GST_OBJECT_UNLOCK (self); GST_OBJECT_UNLOCK (self);
} }
static void
gst_vtenc_set_max_frame_delay (GstVTEnc * self, int delay)
{
GST_OBJECT_LOCK (self);
self->max_frame_delay = delay;
if (self->session != NULL)
gst_vtenc_session_configure_max_frame_delay (self, self->session, delay);
GST_OBJECT_UNLOCK (self);
}
static void static void
gst_vtenc_get_property (GObject * obj, guint prop_id, GValue * value, gst_vtenc_get_property (GObject * obj, guint prop_id, GValue * value,
GParamSpec * pspec) GParamSpec * pspec)
@ -672,6 +695,9 @@ gst_vtenc_get_property (GObject * obj, guint prop_id, GValue * value,
self->max_bitrate / 1000, self->bitrate_window)); self->max_bitrate / 1000, self->bitrate_window));
GST_OBJECT_UNLOCK (self); GST_OBJECT_UNLOCK (self);
break; break;
case PROP_MAX_FRAME_DELAY:
g_value_set_int (value, self->max_frame_delay);
break;
case PROP_PRESERVE_ALPHA: case PROP_PRESERVE_ALPHA:
g_value_set_boolean (value, self->preserve_alpha); g_value_set_boolean (value, self->preserve_alpha);
break; break;
@ -725,6 +751,9 @@ gst_vtenc_set_property (GObject * obj, guint prop_id, const GValue * value,
} }
} }
break; break;
case PROP_MAX_FRAME_DELAY:
gst_vtenc_set_max_frame_delay (self, g_value_get_int (value));
break;
case PROP_PRESERVE_ALPHA: case PROP_PRESERVE_ALPHA:
self->preserve_alpha = g_value_get_boolean (value); self->preserve_alpha = g_value_get_boolean (value);
break; break;
@ -1443,11 +1472,10 @@ gst_vtenc_create_session (GstVTEnc * self)
VTCompressionSessionRef session = NULL; VTCompressionSessionRef session = NULL;
CFMutableDictionaryRef encoder_spec = NULL, pb_attrs = NULL; CFMutableDictionaryRef encoder_spec = NULL, pb_attrs = NULL;
OSStatus status; OSStatus status;
#if !HAVE_IOS
const GstVTEncoderDetails *codec_details = const GstVTEncoderDetails *codec_details =
GST_VTENC_CLASS_GET_CODEC_DETAILS (G_OBJECT_GET_CLASS (self)); GST_VTENC_CLASS_GET_CODEC_DETAILS (G_OBJECT_GET_CLASS (self));
#if !HAVE_IOS
/* Apple's M1 hardware encoding fails when provided with an interlaced ProRes source. /* Apple's M1 hardware encoding fails when provided with an interlaced ProRes source.
* It's most likely a bug in VideoToolbox, as no such limitation has been officially mentioned anywhere. * It's most likely a bug in VideoToolbox, as no such limitation has been officially mentioned anywhere.
* For now let's disable HW encoding entirely when such case occurs. */ * For now let's disable HW encoding entirely when such case occurs. */
@ -1523,6 +1551,9 @@ gst_vtenc_create_session (GstVTEnc * self)
self->max_keyframe_interval); self->max_keyframe_interval);
gst_vtenc_session_configure_max_keyframe_interval_duration (self, session, gst_vtenc_session_configure_max_keyframe_interval_duration (self, session,
self->max_keyframe_interval_duration / ((gdouble) GST_SECOND)); self->max_keyframe_interval_duration / ((gdouble) GST_SECOND));
if (codec_details->format_id == GST_kCMVideoCodecType_Some_AppleProRes)
gst_vtenc_session_configure_max_frame_delay (self, session,
self->max_frame_delay);
gst_vtenc_session_configure_bitrate (self, session, self->bitrate); gst_vtenc_session_configure_bitrate (self, session, self->bitrate);
} }
@ -1701,6 +1732,16 @@ gst_vtenc_session_configure_max_keyframe_interval_duration (GstVTEnc * self,
kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, duration); kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, duration);
} }
static void
gst_vtenc_session_configure_max_frame_delay (GstVTEnc * self,
VTCompressionSessionRef session, int delay)
{
if (delay < 0)
delay = kVTUnlimitedFrameDelayCount;
gst_vtenc_session_configure_property_int (self, session,
kVTCompressionPropertyKey_MaxFrameDelayCount, delay);
}
static void static void
gst_vtenc_session_configure_bitrate (GstVTEnc * self, gst_vtenc_session_configure_bitrate (GstVTEnc * self,
VTCompressionSessionRef session, guint bitrate) VTCompressionSessionRef session, guint bitrate)
@ -2079,7 +2120,6 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame)
/* HEVCWithAlpha encoder has a bug where it does not throttle the amount /* HEVCWithAlpha encoder has a bug where it does not throttle the amount
* of input frames queued internally. Other encoders do not have this * of input frames queued internally. Other encoders do not have this
* problem and correctly block until the internal queue has space. * problem and correctly block until the internal queue has space.
* Trying to use kVTCompressionPropertyKey_MaxFrameDelayCount does not help.
* When paired with a fast enough source like videotestsrc, this can result in * When paired with a fast enough source like videotestsrc, this can result in
* a ton of memory being taken up by frames inside the encoder, eventually killing * a ton of memory being taken up by frames inside the encoder, eventually killing
* the process because of OOM. * the process because of OOM.

View file

@ -84,6 +84,7 @@ struct _GstVTEnc
gdouble quality; gdouble quality;
gint max_keyframe_interval; gint max_keyframe_interval;
GstClockTime max_keyframe_interval_duration; GstClockTime max_keyframe_interval_duration;
gint max_frame_delay;
gint latency_frames; gint latency_frames;
gboolean preserve_alpha; gboolean preserve_alpha;
GstVtencRateControl rate_control; GstVtencRateControl rate_control;