rtpvp8: fix bitstream parsing using the wrong kind of bitreader

VP8 uses a probabilistic bool coder, not a straight bit coder.
This fixes parsing when error-resilient is set.

This commit includes a copy of libvpx's bool coder, BSD licensed.

https://bugzilla.gnome.org/show_bug.cgi?id=652694

gst/rtp/dboolhuff.c

@@ -0,0 +1,68 @@
+/*
+ *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the dboolhuff.LICENSE file in this directory.
+ *  See the libvpx original distribution for more information,
+ *  including patent information, and author information.
+ */
+
+
+#include "dboolhuff.h"
+
+const unsigned char vp8_norm[256] __attribute__ ((aligned (16))) = {
+0, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4,
+      3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+      2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+      2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+int
+vp8dx_start_decode (BOOL_DECODER * br,
+    const unsigned char *source, unsigned int source_sz)
+{
+  br->user_buffer_end = source + source_sz;
+  br->user_buffer = source;
+  br->value = 0;
+  br->count = -8;
+  br->range = 255;
+
+  if (source_sz && !source)
+    return 1;
+
+  /* Populate the buffer */
+  vp8dx_bool_decoder_fill (br);
+
+  return 0;
+}
+
+
+void
+vp8dx_bool_decoder_fill (BOOL_DECODER * br)
+{
+  const unsigned char *bufptr;
+  const unsigned char *bufend;
+  VP8_BD_VALUE value;
+  int count;
+  bufend = br->user_buffer_end;
+  bufptr = br->user_buffer;
+  value = br->value;
+  count = br->count;
+
+  VP8DX_BOOL_DECODER_FILL (count, value, bufptr, bufend);
+
+  br->user_buffer = bufptr;
+  br->value = value;
+  br->count = count;
+}

gst/rtp/dboolhuff.h

@@ -0,0 +1,151 @@
+/*
+ *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the dboolhuff.LICENSE file in this directory.
+ *  See the libvpx original distribution for more information,
+ *  including patent information, and author information.
+ */
+
+
+#ifndef DBOOLHUFF_H
+#define DBOOLHUFF_H
+#include <stddef.h>
+#include <limits.h>
+#include <glib.h>
+
+typedef size_t VP8_BD_VALUE;
+
+# define VP8_BD_VALUE_SIZE ((int)sizeof(VP8_BD_VALUE)*CHAR_BIT)
+/*This is meant to be a large, positive constant that can still be efficiently
+   loaded as an immediate (on platforms like ARM, for example).
+  Even relatively modest values like 100 would work fine.*/
+# define VP8_LOTS_OF_BITS (0x40000000)
+
+typedef struct
+{
+    const unsigned char *user_buffer_end;
+    const unsigned char *user_buffer;
+    VP8_BD_VALUE         value;
+    int                  count;
+    unsigned int         range;
+} BOOL_DECODER;
+
+extern const unsigned char vp8_norm[256] __attribute__((aligned(16)));
+
+int vp8dx_start_decode(BOOL_DECODER *br,
+                       const unsigned char *source,
+                       unsigned int source_sz);
+
+void vp8dx_bool_decoder_fill(BOOL_DECODER *br);
+
+/*The refill loop is used in several places, so define it in a macro to make
+   sure they're all consistent.
+  An inline function would be cleaner, but has a significant penalty, because
+   multiple BOOL_DECODER fields must be modified, and the compiler is not smart
+   enough to eliminate the stores to those fields and the subsequent reloads
+   from them when inlining the function.*/
+#define VP8DX_BOOL_DECODER_FILL(_count,_value,_bufptr,_bufend) \
+    do \
+    { \
+        int shift = VP8_BD_VALUE_SIZE - 8 - ((_count) + 8); \
+        int loop_end, x; \
+        size_t bits_left = ((_bufend)-(_bufptr))*CHAR_BIT; \
+        \
+        x = shift + CHAR_BIT - bits_left; \
+        loop_end = 0; \
+        if(x >= 0) \
+        { \
+            (_count) += VP8_LOTS_OF_BITS; \
+            loop_end = x; \
+            if(!bits_left) break; \
+        } \
+        while(shift >= loop_end) \
+        { \
+            (_count) += CHAR_BIT; \
+            (_value) |= (VP8_BD_VALUE)*(_bufptr)++ << shift; \
+            shift -= CHAR_BIT; \
+        } \
+    } \
+    while(0) \
+
+
+static int vp8dx_decode_bool(BOOL_DECODER *br, int probability) {
+    unsigned int bit = 0;
+    VP8_BD_VALUE value;
+    unsigned int split;
+    VP8_BD_VALUE bigsplit;
+    int count;
+    unsigned int range;
+
+    split = 1 + (((br->range - 1) * probability) >> 8);
+
+    if(br->count < 0)
+        vp8dx_bool_decoder_fill(br);
+
+    value = br->value;
+    count = br->count;
+
+    bigsplit = (VP8_BD_VALUE)split << (VP8_BD_VALUE_SIZE - 8);
+
+    range = split;
+
+    if (value >= bigsplit)
+    {
+        range = br->range - split;
+        value = value - bigsplit;
+        bit = 1;
+    }
+
+    {
+        register unsigned int shift = vp8_norm[range];
+        range <<= shift;
+        value <<= shift;
+        count -= shift;
+    }
+    br->value = value;
+    br->count = count;
+    br->range = range;
+
+    return bit;
+}
+
+static G_GNUC_UNUSED int vp8_decode_value(BOOL_DECODER *br, int bits)
+{
+    int z = 0;
+    int bit;
+
+    for (bit = bits - 1; bit >= 0; bit--)
+    {
+        z |= (vp8dx_decode_bool(br, 0x80) << bit);
+    }
+
+    return z;
+}
+
+static G_GNUC_UNUSED int vp8dx_bool_error(BOOL_DECODER *br)
+{
+    /* Check if we have reached the end of the buffer.
+     *
+     * Variable 'count' stores the number of bits in the 'value' buffer, minus
+     * 8. The top byte is part of the algorithm, and the remainder is buffered
+     * to be shifted into it. So if count == 8, the top 16 bits of 'value' are
+     * occupied, 8 for the algorithm and 8 in the buffer.
+     *
+     * When reading a byte from the user's buffer, count is filled with 8 and
+     * one byte is filled into the value buffer. When we reach the end of the
+     * data, count is additionally filled with VP8_LOTS_OF_BITS. So when
+     * count == VP8_LOTS_OF_BITS - 1, the user's data has been exhausted.
+     */
+    if ((br->count > VP8_BD_VALUE_SIZE) && (br->count < VP8_LOTS_OF_BITS))
+    {
+       /* We have tried to decode bits after the end of
+        * stream was encountered.
+        */
+        return 1;
+    }
+
+    /* No error. */
+    return 0;
+}
+#endif

gst/rtp/gstrtpvp8pay.c

@@ -25,6 +25,7 @@
 #include <gst/base/gstbitreader.h>
 #include <gst/rtp/gstrtppayloads.h>
 #include <gst/rtp/gstrtpbuffer.h>
+#include "dboolhuff.h"
 #include "gstrtpvp8pay.h"
 
 #define FI_FRAG_UNFRAGMENTED 0x0
@@ -130,6 +131,8 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer)
   guint8 tmp8 = 0;
   guint8 *data;
   guint8 partitions;
+  guint offset;
+  BOOL_DECODER bc;
 
   reader = gst_bit_reader_new_from_buffer (buffer);
 
@@ -150,7 +153,8 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer)
   header_size = data[2] << 11 | data[1] << 3 | (data[0] >> 5);
 
   /* Include the uncompressed data blob in the header */
-  header_size += keyframe ? 10 : 3;
+  offset = keyframe ? 10 : 3;
+  header_size += offset;
 
   if (!gst_bit_reader_skip (reader, 24))
     goto error;
@@ -166,109 +170,81 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer)
     if (!gst_bit_reader_get_bits_uint8 (reader, &tmp8, 8) || tmp8 != 0x2a)
       goto error;
 
-    /* Skip horizontal size code (16 bits) vertical size code (16 bits),
-     * color space (1 bit) and clamping type (1 bit) */
-    if (!gst_bit_reader_skip (reader, 34))
+    /* Skip horizontal size code (16 bits) vertical size code (16 bits) */
+    if (!gst_bit_reader_skip (reader, 32))
       goto error;
   }
 
+  offset = keyframe ? 10 : 3;
+  vp8dx_start_decode (&bc, GST_BUFFER_DATA (buffer) + offset,
+      GST_BUFFER_SIZE (buffer) - offset);
+
+  if (keyframe) {
+    /* color space (1 bit) and clamping type (1 bit) */
+    vp8dx_decode_bool (&bc, 0x80);
+    vp8dx_decode_bool (&bc, 0x80);
+  }
+
   /* segmentation_enabled */
-  if (!gst_bit_reader_get_bits_uint8 (reader, &tmp8, 1))
-    goto error;
-
-  if (tmp8 != 0) {
-    gboolean update_mb_segmentation_map;
-    gboolean update_segment_feature_data;
-
-    if (!gst_bit_reader_get_bits_uint8 (reader, &tmp8, 2))
-      goto error;
-
-    update_mb_segmentation_map = (tmp8 & 0x2) != 0;
-    update_segment_feature_data = (tmp8 & 0x1) != 0;
+  if (vp8dx_decode_bool (&bc, 0x80)) {
+    guint8 update_mb_segmentation_map = vp8dx_decode_bool (&bc, 0x80);
+    guint8 update_segment_feature_data = vp8dx_decode_bool (&bc, 0x80);
 
     if (update_segment_feature_data) {
       /* skip segment feature mode */
-      if (!gst_bit_reader_skip (reader, 1))
-        goto error;
+      vp8dx_decode_bool (&bc, 0x80);
 
-      for (i = 0; i < 4; i++) {
       /* quantizer update */
-        if (!gst_bit_reader_get_bits_uint8 (reader, &tmp8, 1))
-          goto error;
-
-        if (tmp8 != 0) {
-          /* skip quantizer value (7 bits) and sign (1 bit) */
-          if (!gst_bit_reader_skip (reader, 8))
-            goto error;
-        }
-      }
-
       for (i = 0; i < 4; i++) {
-        /* loop filter update */
-        if (!gst_bit_reader_get_bits_uint8 (reader, &tmp8, 1))
-          goto error;
-
-        if (tmp8 != 0) {
-          /* skip lf update value (6 bits) and sign (1 bit) */
-          if (!gst_bit_reader_skip (reader, 7))
-            goto error;
+        /* skip flagged quantizer value (7 bits) and sign (1 bit) */
+        if (vp8dx_decode_bool (&bc, 0x80))
+          vp8_decode_value (&bc, 8);
       }
+
+      /* loop filter update */
+      for (i = 0; i < 4; i++) {
+        /* skip flagged lf update value (6 bits) and sign (1 bit) */
+        if (vp8dx_decode_bool (&bc, 0x80))
+          vp8_decode_value (&bc, 7);
       }
     }
 
     if (update_mb_segmentation_map) {
-      for (i = 0; i < 3; i++) {
       /* segment prob update */
-        if (!gst_bit_reader_get_bits_uint8 (reader, &tmp8, 1))
-          goto error;
-
-        if (tmp8 != 0) {
-          /* skip segment prob */
-          if (!gst_bit_reader_skip (reader, 8))
-            goto error;
-        }
+      for (i = 0; i < 3; i++) {
+        /* skip flagged segment prob */
+        if (vp8dx_decode_bool (&bc, 0x80))
+          vp8_decode_value (&bc, 8);
       }
     }
   }
 
   /* skip filter type (1 bit), loop filter level (6 bits) and
    * sharpness level (3 bits) */
-  if (!gst_bit_reader_skip (reader, 10))
-    goto error;
+  vp8_decode_value (&bc, 1);
+  vp8_decode_value (&bc, 6);
+  vp8_decode_value (&bc, 3);
 
   /* loop_filter_adj_enabled */
-  if (!gst_bit_reader_get_bits_uint8 (reader, &tmp8, 1))
-    goto error;
+  if (vp8dx_decode_bool (&bc, 0x80)) {
 
-  if (tmp8 != 0) {
-    /* loop filter adj enabled */
-
-    /* mode_ref_lf_delta_update */
-    if (!gst_bit_reader_get_bits_uint8 (reader, &tmp8, 1))
-      goto error;
-
-    if (tmp8 != 0) {
-      /* mode_ref_lf_data_update */
-      int i;
+    /* delta update */
+    if (vp8dx_decode_bool (&bc, 0x80)) {
 
       for (i = 0; i < 8; i++) {
         /* 8 updates, 1 bit indicate whether there is one and if follow by a
          * 7 bit update */
-        if (!gst_bit_reader_get_bits_uint8 (reader, &tmp8, 1))
-          goto error;
-
-        if (tmp8 != 0) {
-          /* skip delta magnitude (6 bits) and sign (1 bit) */
-          if (!gst_bit_reader_skip (reader, 7))
-            goto error;
-        }
+        if (vp8dx_decode_bool (&bc, 0x80))
+          vp8_decode_value (&bc, 7);
       }
     }
   }
 
-  if (!gst_bit_reader_get_bits_uint8 (reader, &tmp8, 2))
+  if (vp8dx_bool_error (&bc))
     goto error;
 
+  tmp8 = vp8_decode_value (&bc, 2);
+
   partitions = 1 << tmp8;
 
   /* Check if things are still sensible */