applemedia: improve usage of the VT API

Add timing information to CV samples and pass the GstBuffer
as extra data in the decode function

sys/applemedia/vtapi.c

@@ -58,6 +58,7 @@ gst_vt_api_obtain (GError ** error)
     SYM_SPEC (VTDecompressionSessionDecodeFrame),
     SYM_SPEC (VTDecompressionSessionInvalidate),
     SYM_SPEC (VTDecompressionSessionWaitForAsynchronousFrames),
+    SYM_SPEC (VTDecompressionSessionFinishDelayedFrames),
 
     SYM_SPEC (kVTCompressionPropertyKey_AllowTemporalCompression),
     SYM_SPEC (kVTCompressionPropertyKey_AverageDataRate),

sys/applemedia/vtapi.h

@@ -29,6 +29,8 @@ typedef struct _GstVTApi GstVTApi;
 typedef struct _GstVTApiClass GstVTApiClass;
 
 typedef enum _VTStatus VTStatus;
+typedef enum _VTDecodeFrameFlags VTDecodeFrameFlags;
+typedef enum _VTDecodeInfoFlags VTDecodeInfoFlags;
 
 typedef guint32 VTFormatId;
 
@@ -39,8 +41,9 @@ typedef struct _VTDecompressionOutputCallback VTDecompressionOutputCallback;
 
 typedef VTStatus (* VTCompressionOutputCallbackFunc) (void * data, int a2,
     int a3, int a4, CMSampleBufferRef sbuf, int a6, int a7);
-typedef void (* VTDecompressionOutputCallbackFunc) (void * data, gsize unk1,
-    VTStatus result, gsize unk2, CVBufferRef cvbuf);
+typedef void (* VTDecompressionOutputCallbackFunc) (void *data1, void *data2,
+    VTStatus result, VTDecodeInfoFlags info, CVBufferRef cvbuf,
+    CMTime pts, CMTime dts);
 
 enum _VTStatus
 {
@@ -54,6 +57,23 @@ enum _VTFormat
   kVTFormatJPEG = 'jpeg'
 };
 
+enum _VTDecodeFrameFlags
+{
+  kVTDecodeFrame_EnableAsynchronousDecompression = 1<<0,
+  kVTDecodeFrame_DoNotOutputFrame = 1<<1,
+  /* low-power mode that can not decode faster than 1x realtime. */
+  kVTDecodeFrame_1xRealTimePlayback = 1<<2,
+  /* Output frame in PTS order.
+   * Needs to call VTDecompressionSessionFinishDelayedFrames to dequeue */
+  kVTDecodeFrame_EnableTemporalProcessing = 1<<3,
+};
+
+enum _VTDecodeInfoFlags
+{
+  kVTDecodeInfo_Asynchronous = 1UL << 0,
+  kVTDecodeInfo_FrameDropped = 1UL << 1,
+};
+
 struct _VTCompressionOutputCallback
 {
   VTCompressionOutputCallbackFunc func;
@@ -101,12 +121,15 @@ struct _GstVTApi
       VTDecompressionOutputCallback * outputCallback,
       VTDecompressionSessionRef * session);
   VTStatus (* VTDecompressionSessionDecodeFrame)
-      (VTDecompressionSessionRef session, CMSampleBufferRef sbuf, gsize unk1,
-      gsize unk2, gsize unk3);
+      (VTDecompressionSessionRef session, CMSampleBufferRef sbuf,
+       VTDecodeFrameFlags decode_flags, void *src_buf,
+       VTDecodeInfoFlags *info_flags);
   void (* VTDecompressionSessionInvalidate)
       (VTDecompressionSessionRef session);
   VTStatus (* VTDecompressionSessionWaitForAsynchronousFrames)
       (VTDecompressionSessionRef session);
+  VTStatus (* VTDecompressionSessionFinishDelayedFrames)
+      (VTDecompressionSessionRef session);
 
   CFStringRef * kVTCompressionPropertyKey_AllowTemporalCompression;
   CFStringRef * kVTCompressionPropertyKey_AverageDataRate;

sys/applemedia/vtdec.c

@@ -46,8 +46,8 @@ static VTDecompressionSessionRef gst_vtdec_create_session (GstVTDec * self,
 static void gst_vtdec_destroy_session (GstVTDec * self,
     VTDecompressionSessionRef * session);
 static GstFlowReturn gst_vtdec_decode_buffer (GstVTDec * self, GstBuffer * buf);
-static void gst_vtdec_enqueue_frame (void *data, gsize unk1, VTStatus result,
-    gsize unk2, CVBufferRef cvbuf);
+static void gst_vtdec_enqueue_frame (void *data1, void *data2, VTStatus result,
+    VTDecodeInfoFlags info, CVBufferRef cvbuf, CMTime pts, CMTime dts);
 static gboolean gst_vtdec_sink_event (GstPad * pad, GstObject * parent,
     GstEvent * event);
 
@@ -461,13 +461,14 @@ gst_vtdec_decode_buffer (GstVTDec * self, GstBuffer * buf)
   GstVTApi *vt = self->ctx->vt;
   CMSampleBufferRef sbuf;
   VTStatus status;
+  VTDecodeFrameFlags frame_flags = 0;
   GstFlowReturn ret = GST_FLOW_OK;
   guint i;
 
-  self->cur_inbuf = buf;
   sbuf = gst_vtdec_sample_buffer_from (self, buf);
 
-  status = vt->VTDecompressionSessionDecodeFrame (self->session, sbuf, 0, 0, 0);
+  status = vt->VTDecompressionSessionDecodeFrame (self->session, sbuf,
+      frame_flags, buf, NULL);
   if (status != 0) {
     GST_WARNING_OBJECT (self, "VTDecompressionSessionDecodeFrame returned %d",
         status);
@@ -480,7 +481,6 @@ gst_vtdec_decode_buffer (GstVTDec * self, GstBuffer * buf)
   }
 
   CFRelease (sbuf);
-  self->cur_inbuf = NULL;
   gst_buffer_unref (buf);
 
   if (self->cur_outbufs->len > 0) {
@@ -503,10 +503,11 @@ gst_vtdec_decode_buffer (GstVTDec * self, GstBuffer * buf)
 }
 
 static void
-gst_vtdec_enqueue_frame (void *data, gsize unk1, VTStatus result, gsize unk2,
-    CVBufferRef cvbuf)
+gst_vtdec_enqueue_frame (void *data1, void *data2, VTStatus result,
+    VTDecodeInfoFlags info, CVBufferRef cvbuf, CMTime pts, CMTime duration)
 {
-  GstVTDec *self = GST_VTDEC_CAST (data);
+  GstVTDec *self = GST_VTDEC_CAST (data1);
+  GstBuffer *src_buf = GST_BUFFER (data2);
   GstBuffer *buf;
 
   if (result != kVTSuccess)
@@ -528,6 +529,8 @@ gst_vtdec_sample_buffer_from (GstVTDec * self, GstBuffer * buf)
   CMBlockBufferRef bbuf = NULL;
   CMSampleBufferRef sbuf = NULL;
   GstMapInfo map;
+  CMSampleTimingInfo sample_timing;
+  CMSampleTimingInfo time_array[1];
 
   g_assert (self->fmt_desc != NULL);
 
@@ -542,8 +545,13 @@ gst_vtdec_sample_buffer_from (GstVTDec * self, GstBuffer * buf)
   if (status != noErr)
     goto error;
 
+  sample_timing.duration = CMTimeMake (GST_BUFFER_DURATION (buf), 1);
+  sample_timing.presentationTimeStamp = CMTimeMake (GST_BUFFER_PTS (buf), 1);
+  sample_timing.decodeTimeStamp = CMTimeMake (GST_BUFFER_DTS (buf), 1);
+  time_array[0] = sample_timing;
+
   status = CMSampleBufferCreate (NULL, bbuf, TRUE, 0, 0, self->fmt_desc,
-      1, 0, NULL, 0, NULL, &sbuf);
+      1, 1, time_array, 0, NULL, &sbuf);
   if (status != noErr)
     goto error;
 

sys/applemedia/vtdec.h

@@ -67,7 +67,6 @@ struct _GstVTDec
   CMFormatDescriptionRef fmt_desc;
   VTDecompressionSessionRef session;
 
-  GstBuffer * cur_inbuf;
   GPtrArray * cur_outbufs;
 };