From ba8f944df76de4739d0e2d0bec9122708265170b Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Sat, 8 Apr 2023 23:03:15 +0900 Subject: [PATCH] amfencoder: Set output DTS AMF runtime does not provide correct DTS. Although GetPts() seems to be returning DTS, it still needs to be adjusted to meet DTS <= PTS requirement. Do calculate DTS in baseclass instead Part-of: --- .../sys/amfcodec/gstamfav1enc.cpp | 4 +- .../sys/amfcodec/gstamfencoder.cpp | 90 +++++++++++++++---- .../sys/amfcodec/gstamfencoder.h | 3 +- .../sys/amfcodec/gstamfh264enc.cpp | 19 +++- .../sys/amfcodec/gstamfh265enc.cpp | 4 +- 5 files changed, 97 insertions(+), 23 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/amfcodec/gstamfav1enc.cpp b/subprojects/gst-plugins-bad/sys/amfcodec/gstamfav1enc.cpp index dc0b19ba6b..4a8ae6d0a7 100644 --- a/subprojects/gst-plugins-bad/sys/amfcodec/gstamfav1enc.cpp +++ b/subprojects/gst-plugins-bad/sys/amfcodec/gstamfav1enc.cpp @@ -350,7 +350,7 @@ static void gst_amf_av1_enc_set_property (GObject * object, guint prop_id, static void gst_amf_av1_enc_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static gboolean gst_amf_av1_enc_set_format (GstAmfEncoder * encoder, - GstVideoCodecState * state, gpointer component); + GstVideoCodecState * state, gpointer component, guint * num_reorder_frames); static gboolean gst_amf_av1_enc_set_output_state (GstAmfEncoder * encoder, GstVideoCodecState * state, gpointer component); static gboolean gst_amf_av1_enc_set_surface_prop (GstAmfEncoder * encoder, @@ -923,7 +923,7 @@ gst_amf_av1_enc_get_property (GObject * object, guint prop_id, static gboolean gst_amf_av1_enc_set_format (GstAmfEncoder * encoder, - GstVideoCodecState * state, gpointer component) + GstVideoCodecState * state, gpointer component, guint * num_reorder_frames) { GstAmfAv1Enc *self = GST_AMF_AV1_ENC (encoder); GstAmfAv1EncClass *klass = GST_AMF_AV1_ENC_GET_CLASS (self); diff --git a/subprojects/gst-plugins-bad/sys/amfcodec/gstamfencoder.cpp b/subprojects/gst-plugins-bad/sys/amfcodec/gstamfencoder.cpp index ece85adcf9..12fa382fe8 100644 --- a/subprojects/gst-plugins-bad/sys/amfcodec/gstamfencoder.cpp +++ b/subprojects/gst-plugins-bad/sys/amfcodec/gstamfencoder.cpp @@ -29,6 +29,8 @@ #include #include #include +#include +#include /* *INDENT-OFF* */ using namespace Microsoft::WRL; @@ -346,26 +348,32 @@ typedef struct GstMapInfo info; } GstAmfEncoderFrameData; + +/* *INDENT-OFF* */ struct _GstAmfEncoderPrivate { - gint64 adapter_luid; - const wchar_t *codec_id; + gint64 adapter_luid = 0; + const wchar_t *codec_id = nullptr; - GstD3D11Device *device; - GstD3D11Fence *fence; - AMFContext *context; - AMFComponent *comp; - GstBufferPool *internal_pool; + GstD3D11Device *device = nullptr; + GstD3D11Fence *fence = nullptr; + AMFContext *context = nullptr; + AMFComponent *comp = nullptr; + GstBufferPool *internal_pool = nullptr; - GstVideoCodecState *input_state; + GstVideoCodecState *input_state = nullptr; /* High precision clock */ - guint timer_resolution; + guint timer_resolution = 0; + + std::queue timestamp_queue; + GstClockTime dts_offset = 0; + GstClockTime last_dts = GST_CLOCK_TIME_NONE; }; +/* *INDENT-ON* */ #define gst_amf_encoder_parent_class parent_class -G_DEFINE_ABSTRACT_TYPE_WITH_PRIVATE (GstAmfEncoder, gst_amf_encoder, - GST_TYPE_VIDEO_ENCODER); +G_DEFINE_ABSTRACT_TYPE (GstAmfEncoder, gst_amf_encoder, GST_TYPE_VIDEO_ENCODER); static void gst_amf_encoder_dispose (GObject * object); static void gst_amf_encoder_finalize (GObject * object); @@ -440,8 +448,7 @@ gst_amf_encoder_init (GstAmfEncoder * self) GstAmfEncoderPrivate *priv; TIMECAPS time_caps; - priv = self->priv = - (GstAmfEncoderPrivate *) gst_amf_encoder_get_instance_private (self); + priv = self->priv = new GstAmfEncoderPrivate (); gst_video_encoder_set_min_pts (GST_VIDEO_ENCODER (self), GST_SECOND * 60 * 60 * 1000); @@ -477,6 +484,8 @@ gst_amf_encoder_finalize (GObject * object) if (priv->timer_resolution) timeEndPeriod (priv->timer_resolution); + delete priv; + G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -567,12 +576,19 @@ gst_amf_encoder_reset (GstAmfEncoder * self) priv->comp = nullptr; } + std::queue < GstClockTime > empty_queue; + std::swap (priv->timestamp_queue, empty_queue); + + priv->dts_offset = 0; + priv->last_dts = GST_CLOCK_TIME_NONE; + return TRUE; } static GstFlowReturn gst_amf_encoder_process_output (GstAmfEncoder * self, AMFBuffer * buffer) { + GstAmfEncoderPrivate *priv = self->priv; GstAmfEncoderClass *klass = GST_AMF_ENCODER_GET_CLASS (self); GstVideoEncoder *venc = GST_VIDEO_ENCODER_CAST (self); AMF_RESULT result; @@ -614,11 +630,35 @@ gst_amf_encoder_process_output (GstAmfEncoder * self, AMFBuffer * buffer) GST_BUFFER_FLAG_SET (output_buffer, GST_BUFFER_FLAG_MARKER); if (frame) { + GstClockTime dts = GST_CLOCK_TIME_NONE; + + if (GST_CLOCK_TIME_IS_VALID (frame->pts) && !priv->timestamp_queue.empty ()) { + dts = priv->timestamp_queue.front (); + priv->timestamp_queue.pop (); + + if (priv->dts_offset > 0) + dts -= priv->dts_offset; + + if (!GST_CLOCK_TIME_IS_VALID (priv->last_dts)) { + dts = MIN (dts, frame->pts); + priv->last_dts = dts; + } else { + dts = MAX (priv->last_dts, dts); + dts = MIN (dts, frame->pts); + priv->last_dts = dts; + } + } + + frame->dts = dts; frame->output_buffer = output_buffer; if (sync_point) GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (frame); } else { + GstClockTime pts = buffer->GetPts () * 100; + + GST_BUFFER_PTS (output_buffer) = pts; + if (!sync_point) GST_BUFFER_FLAG_SET (output_buffer, GST_BUFFER_FLAG_DELTA_UNIT); @@ -805,6 +845,7 @@ gst_amf_encoder_open_component (GstAmfEncoder * self) AMFFactory *factory = (AMFFactory *) gst_amf_get_factory (); AMFComponentPtr comp; AMF_RESULT result; + guint num_reorder_frames = 0; gst_amf_encoder_drain (self, FALSE); @@ -818,7 +859,8 @@ gst_amf_encoder_open_component (GstAmfEncoder * self) return FALSE; } - if (!klass->set_format (self, priv->input_state, comp.GetPtr ())) { + if (!klass->set_format (self, priv->input_state, comp.GetPtr (), + &num_reorder_frames)) { GST_ERROR_OBJECT (self, "Failed to set format"); return FALSE; } @@ -830,6 +872,19 @@ gst_amf_encoder_open_component (GstAmfEncoder * self) priv->comp = comp.Detach (); + if (num_reorder_frames > 0) { + gint fps_n = 25; + gint fps_d = 1; + + if (priv->input_state->info.fps_n > 0 && priv->input_state->info.fps_d > 0) { + fps_n = priv->input_state->info.fps_n; + fps_d = priv->input_state->info.fps_d; + } + + priv->dts_offset = gst_util_uint64_scale (GST_SECOND, fps_d, fps_n) * + num_reorder_frames; + } + return TRUE; } @@ -1110,7 +1165,8 @@ gst_amf_frame_data_free (GstAmfEncoderFrameData * data) } static GstFlowReturn -gst_amf_encoder_submit_input (GstAmfEncoder * self, AMFSurface * surface) +gst_amf_encoder_submit_input (GstAmfEncoder * self, GstVideoCodecFrame * frame, + AMFSurface * surface) { GstAmfEncoderPrivate *priv = self->priv; AMF_RESULT result; @@ -1122,6 +1178,8 @@ gst_amf_encoder_submit_input (GstAmfEncoder * self, AMFSurface * surface) GST_TRACE_OBJECT (self, "SubmitInput returned %" GST_AMF_RESULT_FORMAT, GST_AMF_RESULT_ARGS (result)); ret = GST_FLOW_OK; + if (GST_CLOCK_TIME_IS_VALID (frame->pts)) + priv->timestamp_queue.push (frame->pts); break; } @@ -1228,7 +1286,7 @@ gst_amf_encoder_handle_frame (GstVideoEncoder * encoder, klass->set_surface_prop (self, frame, surface.GetPtr ()); gst_video_codec_frame_unref (frame); - ret = gst_amf_encoder_submit_input (self, surface.GetPtr ()); + ret = gst_amf_encoder_submit_input (self, frame, surface.GetPtr ()); if (ret == GST_FLOW_OK) ret = gst_amf_encoder_try_output (self, FALSE); if (ret == GST_AMF_ENCODER_FLOW_TRY_AGAIN) diff --git a/subprojects/gst-plugins-bad/sys/amfcodec/gstamfencoder.h b/subprojects/gst-plugins-bad/sys/amfcodec/gstamfencoder.h index 3e72fc0d37..1c9ac943ef 100644 --- a/subprojects/gst-plugins-bad/sys/amfcodec/gstamfencoder.h +++ b/subprojects/gst-plugins-bad/sys/amfcodec/gstamfencoder.h @@ -88,7 +88,8 @@ struct _GstAmfEncoderClass GstAmfEncoderPASupportedOptions pa_supported; gboolean (*set_format) (GstAmfEncoder * encoder, GstVideoCodecState * state, - gpointer component); + gpointer component, + guint * num_reorder_frames); gboolean (*set_output_state) (GstAmfEncoder * encoder, GstVideoCodecState * state, diff --git a/subprojects/gst-plugins-bad/sys/amfcodec/gstamfh264enc.cpp b/subprojects/gst-plugins-bad/sys/amfcodec/gstamfh264enc.cpp index 4e7159dfc4..31966f27b1 100644 --- a/subprojects/gst-plugins-bad/sys/amfcodec/gstamfh264enc.cpp +++ b/subprojects/gst-plugins-bad/sys/amfcodec/gstamfh264enc.cpp @@ -395,7 +395,7 @@ static void gst_amf_h264_enc_get_property (GObject * object, guint prop_id, static GstCaps *gst_amf_h264_enc_getcaps (GstVideoEncoder * encoder, GstCaps * filter); static gboolean gst_amf_h264_enc_set_format (GstAmfEncoder * encoder, - GstVideoCodecState * state, gpointer component); + GstVideoCodecState * state, gpointer component, guint * num_reorder_frames); static gboolean gst_amf_h264_enc_set_output_state (GstAmfEncoder * encoder, GstVideoCodecState * state, gpointer component); static gboolean gst_amf_h264_enc_set_surface_prop (GstAmfEncoder * encoder, @@ -1162,7 +1162,7 @@ gst_amf_h264_enc_getcaps (GstVideoEncoder * encoder, GstCaps * filter) static gboolean gst_amf_h264_enc_set_format (GstAmfEncoder * encoder, - GstVideoCodecState * state, gpointer component) + GstVideoCodecState * state, gpointer component, guint * num_reorder_frames) { GstAmfH264Enc *self = GST_AMF_H264_ENC (encoder); GstAmfH264EncClass *klass = GST_AMF_H264_ENC_GET_CLASS (self); @@ -1501,6 +1501,21 @@ gst_amf_h264_enc_set_format (GstAmfEncoder * encoder, goto error; } + if (dev_caps->bframes && (profile == AMF_VIDEO_ENCODER_PROFILE_MAIN || + profile == AMF_VIDEO_ENCODER_PROFILE_HIGH)) { + int64_val = 0; + result = comp->GetProperty (AMF_VIDEO_ENCODER_B_PIC_PATTERN, &int64_val); + if (result != AMF_OK) { + GST_ERROR_OBJECT (self, "Couldn't get b-frame setting, result %" + GST_AMF_RESULT_FORMAT, GST_AMF_RESULT_ARGS (result)); + goto error; + } + + if (int64_val > 0) { + *num_reorder_frames = (guint) int64_val; + } + } + self->property_updated = FALSE; g_mutex_unlock (&self->prop_lock); diff --git a/subprojects/gst-plugins-bad/sys/amfcodec/gstamfh265enc.cpp b/subprojects/gst-plugins-bad/sys/amfcodec/gstamfh265enc.cpp index 35839ee665..808cb77153 100644 --- a/subprojects/gst-plugins-bad/sys/amfcodec/gstamfh265enc.cpp +++ b/subprojects/gst-plugins-bad/sys/amfcodec/gstamfh265enc.cpp @@ -365,7 +365,7 @@ static void gst_amf_h265_enc_get_property (GObject * object, guint prop_id, static GstCaps *gst_amf_h265_enc_getcaps (GstVideoEncoder * encoder, GstCaps * filter); static gboolean gst_amf_h265_enc_set_format (GstAmfEncoder * encoder, - GstVideoCodecState * state, gpointer component); + GstVideoCodecState * state, gpointer component, guint * num_reorder_frames); static gboolean gst_amf_h265_enc_set_output_state (GstAmfEncoder * encoder, GstVideoCodecState * state, gpointer component); static gboolean gst_amf_h265_enc_set_surface_prop (GstAmfEncoder * encoder, @@ -1054,7 +1054,7 @@ gst_amf_h265_enc_getcaps (GstVideoEncoder * encoder, GstCaps * filter) static gboolean gst_amf_h265_enc_set_format (GstAmfEncoder * encoder, - GstVideoCodecState * state, gpointer component) + GstVideoCodecState * state, gpointer component, guint * num_reorder_frames) { GstAmfH265Enc *self = GST_AMF_H265_ENC (encoder); GstAmfH265EncClass *klass = GST_AMF_H265_ENC_GET_CLASS (self);