vkencoder-private: implement callback to chain codec specific structures

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/8007>

subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.c

@@ -37,6 +37,10 @@ struct _GstVulkanEncoderPrivate
 
   GstCaps *profile_caps;
 
+  GstVulkanEncoderCallbacks callbacks;
+  gpointer callbacks_user_data;
+  GDestroyNotify callbacks_notify;
+
   GstVulkanOperation *exec;
 
   GstVulkanVideoSession session;
@@ -106,6 +110,14 @@ static void
 gst_vulkan_encoder_finalize (GObject * object)
 {
   GstVulkanEncoder *self = GST_VULKAN_ENCODER (object);
+  GstVulkanEncoderPrivate *priv =
+      gst_vulkan_encoder_get_instance_private (self);
+
+  if (priv->callbacks_user_data && priv->callbacks_notify) {
+    priv->callbacks_notify (priv->callbacks_user_data);
+    priv->callbacks_user_data = NULL;
+    priv->callbacks_notify = NULL;
+  }
 
   gst_clear_object (&self->queue);
 
@@ -1066,7 +1078,7 @@ gst_vulkan_encoder_encode (GstVulkanEncoder * self, GstVideoInfo * info,
   };
   pic->dpb_slot = (VkVideoReferenceSlotInfoKHR) {
     .sType = VK_STRUCTURE_TYPE_VIDEO_REFERENCE_SLOT_INFO_KHR,
-    .pNext = pic->codec_dpb_slot_info,
+    .pNext = NULL, /* to fill in callback */
     .slotIndex = slot_index,
     .pPictureResource = &pic->dpb,
   };
@@ -1110,7 +1122,7 @@ gst_vulkan_encoder_encode (GstVulkanEncoder * self, GstVideoInfo * info,
   /* *INDENT-OFF* */
   encode_info = (VkVideoEncodeInfoKHR) {
     .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_INFO_KHR,
-    .pNext = pic->codec_pic_info,
+    .pNext = NULL, /* to fill in callback */
     .flags = 0x0,
     .dstBuffer = ((GstVulkanBufferMemory *) mem)->buffer,
     .dstBufferOffset = pic->offset,
@@ -1137,6 +1149,10 @@ gst_vulkan_encoder_encode (GstVulkanEncoder * self, GstVideoInfo * info,
   encode_info.dstBufferRange = GST_ROUND_DOWN_N (encode_info.dstBufferRange,
       priv->caps.caps.minBitstreamBufferSizeAlignment);
 
+  g_assert (priv->callbacks.setup_codec_pic);
+  priv->callbacks.setup_codec_pic (pic, &encode_info,
+      priv->callbacks_user_data);
+
   gst_vulkan_operation_add_dependency_frame (priv->exec, pic->in_buffer,
       VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
       VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR);
@@ -1277,3 +1293,21 @@ gst_vulkan_encoder_create_from_queue (GstVulkanQueue * queue, guint codec)
 
   return encoder;
 }
+
+void
+gst_vulkan_encoder_set_callbacks (GstVulkanEncoder * self,
+    GstVulkanEncoderCallbacks * callbacks, gpointer user_data,
+    GDestroyNotify notify)
+{
+  GstVulkanEncoderPrivate *priv;
+
+  g_return_if_fail (GST_IS_VULKAN_ENCODER (self) && callbacks);
+
+  priv = gst_vulkan_encoder_get_instance_private (self);
+
+  priv->callbacks = *callbacks;
+  if (priv->callbacks_user_data && priv->callbacks_notify)
+    priv->callbacks_notify (priv->callbacks_user_data);
+  priv->callbacks_user_data = user_data;
+  priv->callbacks_notify = notify;
+}

subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.h

@@ -38,6 +38,21 @@ typedef union _GstVulkanEncoderParameters GstVulkanEncoderParameters;
 typedef union _GstVulkanEncoderParametersOverrides GstVulkanEncoderParametersOverrides;
 typedef union _GstVulkanEncoderParametersFeedback GstVulkanEncoderParametersFeedback;
 typedef struct _GstVulkanEncoderPicture GstVulkanEncoderPicture;
+typedef struct _GstVulkaneEncoderCallbacks GstVulkanEncoderCallbacks;
+
+/**
+ * GstVulkaneEncoderCallbacks:
+ * @setup_codec_pic: Called after VkVideoEncodeInfoKHR and
+ *     VkVideoReferenceSlotInfoKHR are filled so they can be chained with the
+ *     specific codec structures. Called in gst_vulkan_encoder_encode().
+ *
+ * See gst_vulkan_encoder_set_callbacks()
+ */
+struct _GstVulkaneEncoderCallbacks
+{
+  void (*setup_codec_pic) (GstVulkanEncoderPicture * pic,
+      VkVideoEncodeInfoKHR * info, gpointer data);
+};
 
 /**
  * GstVulkanEncoderPicture:
@@ -66,8 +81,6 @@ struct _GstVulkanEncoderPicture
   VkVideoReferenceSlotInfoKHR dpb_slot;
 
   gpointer codec_rc_info;
-  gpointer codec_pic_info;
-  gpointer codec_dpb_slot_info;
 };
 
 /**
@@ -138,6 +151,12 @@ GST_VULKAN_API
 GstVulkanEncoder *      gst_vulkan_encoder_create_from_queue    (GstVulkanQueue * queue,
                                                                  guint codec);
 
+GST_VULKAN_API
+void                    gst_vulkan_encoder_set_callbacks        (GstVulkanEncoder * self,
+                                                                 GstVulkanEncoderCallbacks * callbacks,
+                                                                 gpointer user_data,
+                                                                 GDestroyNotify notify);
+
 GST_VULKAN_API
 gboolean                gst_vulkan_encoder_start                (GstVulkanEncoder * self,
                                                                  GstVulkanVideoProfile * profile,

subprojects/gst-plugins-bad/tests/check/libs/vkvideoencodeh264.c

@@ -369,9 +369,41 @@ allocate_frame (GstVulkanEncoder * enc, int width,
   return frame;
 }
 
-#define PICTURE_TYPE(slice_type, is_ref)                                \
-    (slice_type == STD_VIDEO_H264_SLICE_TYPE_I && is_ref) ?    \
-    STD_VIDEO_H264_PICTURE_TYPE_IDR : (StdVideoH264PictureType) slice_type
+#define PICTURE_TYPE(slice_type, is_ref)                                       \
+  (slice_type == STD_VIDEO_H264_SLICE_TYPE_I && is_ref)                        \
+      ? STD_VIDEO_H264_PICTURE_TYPE_IDR                                        \
+      : (StdVideoH264PictureType)slice_type
+
+static void
+setup_codec_pic (GstVulkanEncoderPicture * pic, VkVideoEncodeInfoKHR * info,
+    gpointer data)
+{
+  GstVulkanH264EncodeFrame *frame = (GstVulkanH264EncodeFrame *) pic;
+  GstVulkanVideoCapabilities *enc_caps = data;
+
+  info->pNext = &frame->enc_pic_info;
+  pic->dpb_slot.pNext = &frame->dpb_slot_info;
+
+  {
+    /* *INDENT-OFF* */
+    frame->enc_pic_info = (VkVideoEncodeH264PictureInfoKHR) {
+      .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_PICTURE_INFO_KHR,
+      .pNext = NULL,
+      .naluSliceEntryCount = 1,
+      .pNaluSliceEntries = &frame->slice_info,
+      .pStdPictureInfo = &frame->pic_info,
+      .generatePrefixNalu =
+          (enc_caps->encoder.codec.h264.flags
+           & VK_VIDEO_ENCODE_H264_CAPABILITY_GENERATE_PREFIX_NALU_BIT_KHR),
+    };
+    frame->dpb_slot_info = (VkVideoEncodeH264DpbSlotInfoKHR) {
+      .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_DPB_SLOT_INFO_KHR,
+      .pNext = NULL,
+      .pStdReferenceInfo = &frame->ref_info,
+    };
+    /* *INDENT-ON* */
+  }
+}
 
 static void
 encode_frame (GstVulkanEncoder * enc, GstVulkanH264EncodeFrame * frame,
@@ -383,11 +415,14 @@ encode_frame (GstVulkanEncoder * enc, GstVulkanH264EncodeFrame * frame,
   int i, ref_pics_num = 0;
   GstVulkanEncoderPicture *ref_pics[16] = { NULL, };
   GstVulkanEncoderPicture *picture = &frame->picture;
+  GstVulkanEncoderCallbacks cb = { setup_codec_pic };
 
   GST_DEBUG ("Encoding frame num:%d", frame_num);
 
   fail_unless (gst_vulkan_encoder_caps (enc, &enc_caps));
 
+  gst_vulkan_encoder_set_callbacks (enc, &cb, &enc_caps, NULL);
+
   frame->slice_hdr = (StdVideoEncodeH264SliceHeader) {
     /* *INDENT-OFF* */
     .flags = (StdVideoEncodeH264SliceHeaderFlags) {
@@ -462,16 +497,6 @@ encode_frame (GstVulkanEncoder * enc, GstVulkanH264EncodeFrame * frame,
     .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_RATE_CONTROL_INFO_KHR,
   };
 
-  frame->enc_pic_info = (VkVideoEncodeH264PictureInfoKHR) {
-    .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_PICTURE_INFO_KHR,
-    .pNext = NULL,
-    .naluSliceEntryCount = 1,
-    .pNaluSliceEntries = &frame->slice_info,
-    .pStdPictureInfo = &frame->pic_info,
-    .generatePrefixNalu = (enc_caps.encoder.codec.h264.flags
-         & VK_VIDEO_ENCODE_H264_CAPABILITY_GENERATE_PREFIX_NALU_BIT_KHR),
-  };
-
   frame->ref_info = (StdVideoEncodeH264ReferenceInfo) {
     .flags = {
       .used_for_long_term_reference = 0,
@@ -483,17 +508,9 @@ encode_frame (GstVulkanEncoder * enc, GstVulkanH264EncodeFrame * frame,
     .long_term_frame_idx = 0,
     .temporal_id = 0,
   };
-
-  frame->dpb_slot_info = (VkVideoEncodeH264DpbSlotInfoKHR) {
-    .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_DPB_SLOT_INFO_KHR,
-    .pNext = NULL,
-    .pStdReferenceInfo = &frame->ref_info,
-  };
   /* *INDENT-ON* */
 
-  picture->codec_pic_info = &frame->enc_pic_info;
   picture->codec_rc_info = &frame->rc_info;
-  picture->codec_dpb_slot_info = &frame->dpb_slot_info;
 
   for (i = 0; i < list0_num; i++) {
     ref_pics[i] = &list0[i]->picture;

subprojects/gst-plugins-bad/tests/check/libs/vkvideoencodeh265.c

@@ -383,6 +383,34 @@ allocate_frame (GstVulkanEncoder * enc, int width,
     slice_type == STD_VIDEO_H265_SLICE_TYPE_B ? STD_VIDEO_H265_PICTURE_TYPE_B: \
     (StdVideoH265PictureType) slice_type
 
+static void
+setup_codec_pic (GstVulkanEncoderPicture * pic, VkVideoEncodeInfoKHR * info,
+    gpointer data)
+{
+  GstVulkanH265EncodeFrame *frame = (GstVulkanH265EncodeFrame *) pic;
+
+  info->pNext = &frame->enc_pic_info;
+  pic->dpb_slot.pNext = &frame->dpb_slot_info;
+
+  {
+    /* *INDENT-OFF* */
+    frame->enc_pic_info = (VkVideoEncodeH265PictureInfoKHR) {
+      .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_PICTURE_INFO_KHR,
+      .pNext = NULL,
+      .naluSliceSegmentEntryCount = 1,
+      .pNaluSliceSegmentEntries = &frame->slice_info,
+      .pStdPictureInfo = &frame->pic_info,
+    };
+    frame->dpb_slot_info = (VkVideoEncodeH265DpbSlotInfoKHR) {
+      .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_DPB_SLOT_INFO_KHR,
+      .pNext = NULL,
+      .pStdReferenceInfo = &frame->ref_info,
+    };
+    /* *INDENT-ON* */
+  }
+}
+
+
 static void
 encode_frame (GstVulkanEncoder * enc, GstVulkanH265EncodeFrame * frame,
     StdVideoH265SliceType slice_type, guint frame_num,
@@ -394,10 +422,13 @@ encode_frame (GstVulkanEncoder * enc, GstVulkanH265EncodeFrame * frame,
   GstVulkanEncoderPicture *ref_pics[16] = { NULL, };
   gint16 delta_poc_s0_minus1 = 0, delta_poc_s1_minus1 = 0;
   GstVulkanEncoderPicture *picture = &frame->picture;
-  gint picture_type = PICTURE_TYPE(slice_type, frame->is_ref);
+  gint picture_type = PICTURE_TYPE (slice_type, frame->is_ref);
+  GstVulkanEncoderCallbacks cb = { setup_codec_pic };
 
   GST_DEBUG ("Encoding frame num: %d", frame_num);
 
+  gst_vulkan_encoder_set_callbacks (enc, &cb, NULL, NULL);
+
   ref_pics_num = list0_num + list1_num;
 
   frame->slice_wt = (StdVideoEncodeH265WeightTable) {
@@ -534,14 +565,6 @@ encode_frame (GstVulkanEncoder * enc, GstVulkanH265EncodeFrame * frame,
     .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_RATE_CONTROL_INFO_KHR,
   };
 
-  frame->enc_pic_info = (VkVideoEncodeH265PictureInfoKHR) {
-    .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_PICTURE_INFO_KHR,
-    .pNext = NULL,
-    .naluSliceSegmentEntryCount = 1,
-    .pNaluSliceSegmentEntries = &frame->slice_info,
-    .pStdPictureInfo = &frame->pic_info,
-  };
-
   frame->ref_info = (StdVideoEncodeH265ReferenceInfo) {
     .flags = (StdVideoEncodeH265ReferenceInfoFlags) {
       .used_for_long_term_reference = 0,
@@ -551,17 +574,9 @@ encode_frame (GstVulkanEncoder * enc, GstVulkanH265EncodeFrame * frame,
     .PicOrderCntVal = frame->pic_order_cnt,
     .TemporalId = 0,
   };
-
-  frame->dpb_slot_info = (VkVideoEncodeH265DpbSlotInfoKHR) {
-    .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_DPB_SLOT_INFO_KHR,
-    .pNext = NULL,
-    .pStdReferenceInfo = &frame->ref_info,
-  };
   /* *INDENT-ON* */
 
-  picture->codec_pic_info = &frame->enc_pic_info;
   picture->codec_rc_info = &frame->rc_info;
-  picture->codec_dpb_slot_info = &frame->dpb_slot_info;
 
   for (i = 0; i < list0_num; i++) {
     ref_pics[i] = &list0[i]->picture;