theoraenc: add "dup-on-gap" option

This option will produce duplicate frames if we get
a frame with GAP flag. This will reduce CPU load and file size.

This option should be disabled for real time applications, because it
collects GAP frames and waits until it gets a non GAP frame to start
encoding.

v30.06.2011: make some spell changes.
v03.07.2011: add handling of EOS and discontinuous for dup-on-gap.
v19.12.2011: fix pointer dangling in theora_timefifo_free
v20.12.2010: fix timestamp bug for dup-on-gap=0

Bugzilla: https://bugzilla.gnome.org/show_bug.cgi?id=627459

Signed-off-by: Oleksij Rempel (Alexey Fisher) <bug-track@fisher-privat.net>

ext/theora/gsttheoraenc.c

@@ -130,6 +130,7 @@ _ilog (unsigned int v)
 #define THEORA_DEF_RATE_BUFFER          0
 #define THEORA_DEF_MULTIPASS_CACHE_FILE NULL
 #define THEORA_DEF_MULTIPASS_MODE       MULTIPASS_MODE_SINGLE_PASS
+#define THEORA_DEF_DUP_ON_GAP           FALSE
 enum
 {
   PROP_0,
@@ -152,7 +153,8 @@ enum
   PROP_CAP_UNDERFLOW,
   PROP_RATE_BUFFER,
   PROP_MULTIPASS_CACHE_FILE,
-  PROP_MULTIPASS_MODE
+  PROP_MULTIPASS_MODE,
+  PROP_DUP_ON_GAP
       /* FILL ME */
 };
 
@@ -279,6 +281,11 @@ static gboolean theora_enc_write_multipass_cache (GstTheoraEnc * enc,
 
 static char *theora_enc_get_supported_formats (void);
 
+static void theora_timefifo_free (GstTheoraEnc * enc);
+static GstFlowReturn
+theora_enc_encode_and_push (GstTheoraEnc * enc, ogg_packet op,
+    GstBuffer * buffer);
+
 static void
 gst_theora_enc_base_init (gpointer g_class)
 {
@@ -418,6 +425,16 @@ gst_theora_enc_class_init (GstTheoraEncClass * klass)
           "Single pass or first/second pass", GST_TYPE_MULTIPASS_MODE,
           THEORA_DEF_MULTIPASS_MODE,
           (GParamFlags) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+  g_object_class_install_property (gobject_class, PROP_DUP_ON_GAP,
+      g_param_spec_boolean ("dup-on-gap", "Create DUP frame on GAP flag",
+          "Allow codec to handle frames with GAP flag as duplicates "
+          "of previous frame. "
+          "This is good to work with variable frame rate stabilized "
+          "by videorate element. It will add variable latency with maximal "
+          "size of keyframe distance, this way it is a bad idea "
+          "to use with live streams.",
+          THEORA_DEF_DUP_ON_GAP,
+          (GParamFlags) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
   caps_string = g_strdup_printf ("video/x-raw-yuv, "
       "format = (fourcc) { %s }, "
@@ -463,6 +480,7 @@ gst_theora_enc_init (GstTheoraEnc * enc, GstTheoraEncClass * g_class)
   enc->cap_overflow = THEORA_DEF_CAP_OVERFLOW;
   enc->cap_underflow = THEORA_DEF_CAP_UNDERFLOW;
   enc->rate_buffer = THEORA_DEF_RATE_BUFFER;
+  enc->dup_on_gap = THEORA_DEF_DUP_ON_GAP;
 
   enc->multipass_mode = THEORA_DEF_MULTIPASS_MODE;
   enc->multipass_cache_file = THEORA_DEF_MULTIPASS_CACHE_FILE;
@@ -565,6 +583,8 @@ theora_enc_clear (GstTheoraEnc * enc)
   enc->granulepos_offset = 0;
   enc->timestamp_offset = 0;
 
+  theora_timefifo_free (enc);
+
   enc->next_ts = GST_CLOCK_TIME_NONE;
   enc->next_discont = FALSE;
   enc->expected_ts = GST_CLOCK_TIME_NONE;
@@ -906,6 +926,9 @@ theora_enc_sink_event (GstPad * pad, GstEvent * event)
     }
     case GST_EVENT_EOS:
       if (enc->initialised) {
+        /* clear all standing buffers */
+        if (enc->dup_on_gap)
+          theora_enc_encode_and_push (enc, op, NULL);
         /* push last packet with eos flag, should not be called */
         while (th_encode_packetout (enc->encoder, 1, &op)) {
           GstClockTime next_time =
@@ -927,6 +950,7 @@ theora_enc_sink_event (GstPad * pad, GstEvent * event)
     case GST_EVENT_FLUSH_STOP:
       gst_segment_init (&enc->segment, GST_FORMAT_UNDEFINED);
       res = gst_pad_push_event (enc->srcpad, event);
+      theora_timefifo_free (enc);
       break;
     case GST_EVENT_CUSTOM_DOWNSTREAM:
     {
@@ -1149,18 +1173,193 @@ theora_enc_write_multipass_cache (GstTheoraEnc * enc, gboolean begin,
   return TRUE;
 }
 
+/**
+ * g_slice_free can't be used with g_queue_foreach.
+ * so we create new function with predefined GstClockTime size.
+ */
+static void
+theora_free_gstclocktime (gpointer mem)
+{
+  g_slice_free (GstClockTime, mem);
+}
+
+static void
+theora_timefifo_in (GstTheoraEnc * enc, const GstClockTime * timestamp)
+{
+  GstClockTime *ptr;
+
+  if (!enc->t_queue)
+    enc->t_queue = g_queue_new ();
+
+  g_assert (enc->t_queue != NULL);
+
+  ptr = g_slice_new (GstClockTime);
+  *ptr = *timestamp;
+
+  g_queue_push_head (enc->t_queue, ptr);
+}
+
+static GstClockTime
+theora_timefifo_out (GstTheoraEnc * enc)
+{
+  GstClockTime ret, *ptr;
+
+  g_assert (enc->t_queue != NULL);
+
+  ptr = g_queue_pop_tail (enc->t_queue);
+  g_assert (ptr != NULL);
+
+  ret = *ptr;
+  theora_free_gstclocktime (ptr);
+
+  return ret;
+}
+
+/**
+ * theora_timefifo_truncate - truncate the timestamp queue.
+ * After frame encoding we should have only one buffer for next time.
+ * The count of timestamps should be the same. If it is less,
+ * some thing really bad has happened. If it is bigger, encoder
+ * decided to return less then we ordered.
+ * TODO: for now we will just drop this timestamps. The better solution
+ * probably will be to recovery frames by recovery timestamps with
+ * last buffer.
+ */
+static void
+theora_timefifo_truncate (GstTheoraEnc * enc)
+{
+  if (enc->dup_on_gap) {
+    guint length;
+    g_assert (enc->t_queue != NULL);
+    length = g_queue_get_length (enc->t_queue);
+
+    if (length > 1) {
+      /* it is also not good if we have more then 1. */
+      GST_DEBUG_OBJECT (enc, "Dropping %u time stamps", length - 1);
+      while (g_queue_get_length (enc->t_queue) > 1) {
+        theora_timefifo_out (enc);
+      }
+    }
+  }
+}
+
+static void
+theora_timefifo_free (GstTheoraEnc * enc)
+{
+  if (enc->t_queue) {
+    if (g_queue_get_length (enc->t_queue))
+      g_queue_foreach (enc->t_queue, (GFunc) theora_free_gstclocktime, NULL);
+    g_queue_free (enc->t_queue);
+    enc->t_queue = NULL;
+  }
+  /* prevbuf makes no sense without timestamps,
+   * so clear it too. */
+  if (enc->prevbuf) {
+    gst_buffer_unref (enc->prevbuf);
+    enc->prevbuf = NULL;
+  }
+
+}
+
+static void
+theora_update_prevbuf (GstTheoraEnc * enc, GstBuffer * buffer)
+{
+  if (enc->prevbuf) {
+    gst_buffer_unref (enc->prevbuf);
+    enc->prevbuf = NULL;
+  }
+  enc->prevbuf = gst_buffer_ref (buffer);
+}
+
+/**
+ * theora_enc_encode_and_push - encode buffer or queued previous buffer
+ * buffer - buffer to encode. If set to NULL it should encode only
+ *          queued buffers and produce dups if needed.
+ */
+
 static GstFlowReturn
 theora_enc_encode_and_push (GstTheoraEnc * enc, ogg_packet op,
-    GstClockTime timestamp, GstClockTime running_time,
+    GstBuffer * buffer)
-    GstClockTime duration, GstBuffer * buffer)
 {
   GstFlowReturn ret;
   th_ycbcr_buffer ycbcr;
   gint res;
 
+  if (enc->dup_on_gap) {
+    guint t_queue_length;
+
+    if (enc->t_queue)
+      t_queue_length = g_queue_get_length (enc->t_queue);
+    else
+      t_queue_length = 0;
+
+    if (buffer) {
+      GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buffer);
+
+      /* videorate can easy create 200 dup frames in one shot.
+       * In this case th_encode_ctl will just return TH_EINVAL
+       * and we will generate only one frame as result.
+       * To make us more bullet proof, make sure we have no
+       * more dup frames than keyframe interval.
+       */
+      if (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_GAP) &&
+          enc->keyframe_force > t_queue_length) {
+        GST_DEBUG_OBJECT (enc, "Got GAP frame, queue as duplicate.");
+
+        theora_timefifo_in (enc, &timestamp);
+        gst_buffer_unref (buffer);
+        return GST_FLOW_OK;
+      } else {
+        theora_timefifo_in (enc, &timestamp);
+        /* We should have one frame delay to create correct frame order.
+         * First time we got buffer, prevbuf should be empty. Nothing else
+         * should be done here.
+         */
+        if (!enc->prevbuf) {
+          theora_update_prevbuf (enc, buffer);
+          gst_buffer_unref (buffer);
+          return GST_FLOW_OK;
+        } else {
+          theora_update_prevbuf (enc, buffer);
+          /* after theora_update_prevbuf t_queue_length was changed */
+          t_queue_length++;
+
+          if (t_queue_length > 2) {
+            /* now in t_queue_length should be two real buffers: current and
+             * previous. All others are timestamps of duplicate frames. */
+            t_queue_length -= 2;
+            res = th_encode_ctl (enc->encoder, TH_ENCCTL_SET_DUP_COUNT,
+                &t_queue_length, sizeof (t_queue_length));
+            if (res < 0)
+              GST_WARNING_OBJECT (enc, "Failed marking dups for last frame");
+          }
+        }
+      }
+    } else {
+      /* if there is no buffer, then probably we got EOS or discontinuous.
+       * We need to encode every thing what was left in the queue
+       */
+      GST_DEBUG_OBJECT (enc, "Encode collected buffers.");
+      if (t_queue_length > 1) {
+        t_queue_length--;
+        res = th_encode_ctl (enc->encoder, TH_ENCCTL_SET_DUP_COUNT,
+            &t_queue_length, sizeof (t_queue_length));
+        if (res < 0)
+          GST_WARNING_OBJECT (enc, "Failed marking dups for last frame.");
+      } else {
+        GST_DEBUG_OBJECT (enc, "Prevbuffer is empty. Nothing to encode.");
+        return GST_FLOW_OK;
+      }
+    }
+    theora_enc_init_buffer (ycbcr, &enc->info, GST_BUFFER_DATA (enc->prevbuf));
+  } else
     theora_enc_init_buffer (ycbcr, &enc->info, GST_BUFFER_DATA (buffer));
 
-  if (theora_enc_is_discontinuous (enc, running_time, duration)) {
+  /* check for buffer, it can be optional */
+  if (enc->current_discont && buffer) {
+    GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buffer);
+    GstClockTime running_time =
+        gst_segment_to_running_time (&enc->segment, GST_FORMAT_TIME, timestamp);
     theora_enc_reset (enc);
     enc->granulepos_offset =
         gst_util_uint64_scale (running_time, enc->fps_n,
@@ -1192,19 +1391,32 @@ theora_enc_encode_and_push (GstTheoraEnc * enc, ogg_packet op,
 
   ret = GST_FLOW_OK;
   while (th_encode_packetout (enc->encoder, 0, &op)) {
-    GstClockTime next_time;
+    GstClockTime next_time, duration;
+    GstClockTime timestamp = 0;
+    GST_DEBUG_OBJECT (enc, "encoded. granule:%" G_GINT64_FORMAT ", packet:%p, "
+        "bytes:%ld", op.granulepos, op.packet, op.bytes);
 
     next_time = th_granule_time (enc->encoder, op.granulepos) * GST_SECOND;
+    duration = next_time - enc->next_ts;
 
-    ret =
+    if (enc->dup_on_gap && !enc->current_discont)
-        theora_push_packet (enc, &op, timestamp, enc->next_ts,
+      timestamp = theora_timefifo_out (enc);
-        next_time - enc->next_ts);
+    else
+      timestamp = GST_BUFFER_TIMESTAMP (buffer);
+
+    ret = theora_push_packet (enc, &op, timestamp, enc->next_ts, duration);
 
     enc->next_ts = next_time;
-    if (ret != GST_FLOW_OK)
+    if (ret != GST_FLOW_OK) {
+      theora_timefifo_truncate (enc);
       goto data_push;
     }
+  }
+
+  theora_timefifo_truncate (enc);
+  if (buffer)
     gst_buffer_unref (buffer);
+  enc->current_discont = FALSE;
 
   return ret;
 
@@ -1374,8 +1586,14 @@ theora_enc_chain (GstPad * pad, GstBuffer * buffer)
     enc->next_ts = 0;
   }
 
-  ret = theora_enc_encode_and_push (enc, op, timestamp, running_time, duration,
+  enc->current_discont = theora_enc_is_discontinuous (enc,
-      buffer);
+      running_time, duration);
+
+  /* empty queue if discontinuous */
+  if (enc->current_discont && enc->dup_on_gap)
+    theora_enc_encode_and_push (enc, op, NULL);
+
+  ret = theora_enc_encode_and_push (enc, op, buffer);
 
   return ret;
 
@@ -1557,6 +1775,9 @@ theora_enc_set_property (GObject * object, guint prop_id,
     case PROP_MULTIPASS_MODE:
       enc->multipass_mode = g_value_get_enum (value);
       break;
+    case PROP_DUP_ON_GAP:
+      enc->dup_on_gap = g_value_get_boolean (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -1634,6 +1855,9 @@ theora_enc_get_property (GObject * object, guint prop_id,
     case PROP_MULTIPASS_MODE:
       g_value_set_enum (value, enc->multipass_mode);
       break;
+    case PROP_DUP_ON_GAP:
+      g_value_set_boolean (value, enc->dup_on_gap);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;

ext/theora/gsttheoraenc.h

@@ -122,6 +122,13 @@ struct _GstTheoraEnc
   gboolean cap_underflow;
   int rate_buffer;
 
+  /* variables for dup-on-gap */
+  gboolean dup_on_gap;
+  gboolean current_discont;
+  GstBuffer *prevbuf;
+  GQueue *t_queue;
+  /* end dup-on-gap */
+
   GstTheoraEncMultipassMode multipass_mode;
   GIOChannel *multipass_cache_fd;
   GstAdapter *multipass_cache_adapter;