diff --git a/ext/dirac/gstdiracenc.cc b/ext/dirac/gstdiracenc.cc
index 7094779d35..6e14fff69d 100644
--- a/ext/dirac/gstdiracenc.cc
+++ b/ext/dirac/gstdiracenc.cc
@@ -1247,7 +1247,7 @@ gst_dirac_enc_shape_output_ogg (GstBaseVideoEncoder * base_video_encoder,
     GST_BUFFER_OFFSET_END (buf) = dirac_enc->last_granulepos;
   }
 
-  gst_buffer_set_caps (buf, base_video_encoder->caps);
+  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC(base_video_encoder)->caps);
 
   return gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 }
@@ -1262,12 +1262,12 @@ gst_dirac_enc_shape_output_quicktime (GstBaseVideoEncoder * base_video_encoder,
   state = gst_base_video_encoder_get_state (base_video_encoder);
 
   GST_BUFFER_TIMESTAMP (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment, frame->presentation_frame_number);
+      &GST_BASE_VIDEO_CODEC(base_video_encoder)->segment, frame->presentation_frame_number);
   GST_BUFFER_DURATION (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment,
+      &GST_BASE_VIDEO_CODEC(base_video_encoder)->segment,
       frame->presentation_frame_number + 1) - GST_BUFFER_TIMESTAMP (buf);
-  GST_BUFFER_OFFSET_END (buf) =
-      gst_video_state_get_timestamp (state, &base_video_encoder->segment,
+  GST_BUFFER_OFFSET_END (buf) = gst_video_state_get_timestamp (state,
+      &GST_BASE_VIDEO_CODEC(base_video_encoder)->segment,
       frame->system_frame_number);
   GST_BUFFER_OFFSET (buf) = GST_CLOCK_TIME_NONE;
 
@@ -1278,7 +1278,7 @@ gst_dirac_enc_shape_output_quicktime (GstBaseVideoEncoder * base_video_encoder,
     GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT);
   }
 
-  gst_buffer_set_caps (buf, base_video_encoder->caps);
+  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC(base_video_encoder)->caps);
 
   return gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 }
@@ -1293,17 +1293,19 @@ gst_dirac_enc_shape_output_mp4 (GstBaseVideoEncoder * base_video_encoder,
   state = gst_base_video_encoder_get_state (base_video_encoder);
 
   GST_BUFFER_TIMESTAMP (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment, frame->presentation_frame_number);
+      &GST_BASE_VIDEO_CODEC(base_video_encoder)->segment,
+      frame->presentation_frame_number);
   GST_BUFFER_DURATION (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment,
+      &GST_BASE_VIDEO_CODEC(base_video_encoder)->segment,
       frame->presentation_frame_number + 1) - GST_BUFFER_TIMESTAMP (buf);
-  GST_BUFFER_OFFSET_END (buf) =
-      gst_video_state_get_timestamp (state, &base_video_encoder->segment,
+  GST_BUFFER_OFFSET_END (buf) = gst_video_state_get_timestamp (state,
+      &GST_BASE_VIDEO_CODEC(base_video_encoder)->segment,
       frame->decode_frame_number);
   GST_BUFFER_OFFSET (buf) = GST_CLOCK_TIME_NONE;
 
   GST_BUFFER_OFFSET_END (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment, frame->system_frame_number);
+      &GST_BASE_VIDEO_CODEC(base_video_encoder)->segment,
+      frame->system_frame_number);
 
   if (frame->is_sync_point &&
       frame->presentation_frame_number == frame->system_frame_number) {
@@ -1312,7 +1314,7 @@ gst_dirac_enc_shape_output_mp4 (GstBaseVideoEncoder * base_video_encoder,
     GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT);
   }
 
-  gst_buffer_set_caps (buf, base_video_encoder->caps);
+  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC(base_video_encoder)->caps);
 
   return gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 }
diff --git a/ext/schroedinger/gstschroenc.c b/ext/schroedinger/gstschroenc.c
index 9b0839dea6..1ede926d8e 100644
--- a/ext/schroedinger/gstschroenc.c
+++ b/ext/schroedinger/gstschroenc.c
@@ -641,7 +641,7 @@ gst_schro_enc_shape_output_ogg (GstBaseVideoEncoder * base_video_encoder,
     GST_BUFFER_OFFSET_END (buf) = schro_enc->last_granulepos;
   }
 
-  gst_buffer_set_caps (buf, base_video_encoder->caps);
+  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
 
   return gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 }
@@ -656,12 +656,14 @@ gst_schro_enc_shape_output_quicktime (GstBaseVideoEncoder * base_video_encoder,
   state = gst_base_video_encoder_get_state (base_video_encoder);
 
   GST_BUFFER_TIMESTAMP (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment, frame->presentation_frame_number);
+      &GST_BASE_VIDEO_CODEC (base_video_encoder)->segment,
+      frame->presentation_frame_number);
   GST_BUFFER_DURATION (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment,
+      &GST_BASE_VIDEO_CODEC (base_video_encoder)->segment,
       frame->presentation_frame_number + 1) - GST_BUFFER_TIMESTAMP (buf);
   GST_BUFFER_OFFSET_END (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment, frame->system_frame_number);
+      &GST_BASE_VIDEO_CODEC (base_video_encoder)->segment,
+      frame->system_frame_number);
   GST_BUFFER_OFFSET (buf) = GST_CLOCK_TIME_NONE;
 
   if (frame->is_sync_point &&
@@ -671,7 +673,7 @@ gst_schro_enc_shape_output_quicktime (GstBaseVideoEncoder * base_video_encoder,
     GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT);
   }
 
-  gst_buffer_set_caps (buf, base_video_encoder->caps);
+  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
 
   return gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 }
@@ -686,16 +688,19 @@ gst_schro_enc_shape_output_mp4 (GstBaseVideoEncoder * base_video_encoder,
   state = gst_base_video_encoder_get_state (base_video_encoder);
 
   GST_BUFFER_TIMESTAMP (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment, frame->presentation_frame_number);
+      &GST_BASE_VIDEO_CODEC (base_video_encoder)->segment,
+      frame->presentation_frame_number);
   GST_BUFFER_DURATION (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment,
+      &GST_BASE_VIDEO_CODEC (base_video_encoder)->segment,
       frame->presentation_frame_number + 1) - GST_BUFFER_TIMESTAMP (buf);
   GST_BUFFER_OFFSET_END (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment, frame->decode_frame_number);
+      &GST_BASE_VIDEO_CODEC (base_video_encoder)->segment,
+      frame->decode_frame_number);
   GST_BUFFER_OFFSET (buf) = GST_CLOCK_TIME_NONE;
 
   GST_BUFFER_OFFSET_END (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment, frame->system_frame_number);
+      &GST_BASE_VIDEO_CODEC (base_video_encoder)->segment,
+      frame->system_frame_number);
 
   if (frame->is_sync_point &&
       frame->presentation_frame_number == frame->system_frame_number) {
@@ -704,7 +709,7 @@ gst_schro_enc_shape_output_mp4 (GstBaseVideoEncoder * base_video_encoder,
     GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DELTA_UNIT);
   }
 
-  gst_buffer_set_caps (buf, base_video_encoder->caps);
+  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
 
   return gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 }
diff --git a/ext/vp8/gstvp8dec.c b/ext/vp8/gstvp8dec.c
index 7bad465eba..ecc75635ee 100644
--- a/ext/vp8/gstvp8dec.c
+++ b/ext/vp8/gstvp8dec.c
@@ -330,21 +330,17 @@ static void
 gst_vp8_dec_image_to_buffer (GstVP8Dec * dec, const vpx_image_t * img,
     GstBuffer * buffer)
 {
-  GstBaseVideoDecoder *decoder = (GstBaseVideoDecoder *) dec;
   int stride, w, h, i;
   guint8 *d;
+  GstVideoState *state = &GST_BASE_VIDEO_CODEC (dec)->state;
 
   d = GST_BUFFER_DATA (buffer) +
-      gst_video_format_get_component_offset (decoder->state.format, 0,
-      decoder->state.width, decoder->state.height);
-  stride =
-      gst_video_format_get_row_stride (decoder->state.format, 0,
-      decoder->state.width);
-  h = gst_video_format_get_component_height (decoder->state.format, 0,
-      decoder->state.height);
+      gst_video_format_get_component_offset (state->format, 0,
+      state->width, state->height);
+  stride = gst_video_format_get_row_stride (state->format, 0, state->width);
+  h = gst_video_format_get_component_height (state->format, 0, state->height);
   h = MIN (h, img->h);
-  w = gst_video_format_get_component_width (decoder->state.format, 0,
-      decoder->state.width);
+  w = gst_video_format_get_component_width (state->format, 0, state->width);
   w = MIN (w, img->w);
 
   for (i = 0; i < h; i++)
@@ -352,24 +348,20 @@ gst_vp8_dec_image_to_buffer (GstVP8Dec * dec, const vpx_image_t * img,
         img->planes[VPX_PLANE_Y] + i * img->stride[VPX_PLANE_Y], w);
 
   d = GST_BUFFER_DATA (buffer) +
-      gst_video_format_get_component_offset (decoder->state.format, 1,
-      decoder->state.width, decoder->state.height);
-  stride =
-      gst_video_format_get_row_stride (decoder->state.format, 1,
-      decoder->state.width);
-  h = gst_video_format_get_component_height (decoder->state.format, 1,
-      decoder->state.height);
+      gst_video_format_get_component_offset (state->format, 1,
+      state->width, state->height);
+  stride = gst_video_format_get_row_stride (state->format, 1, state->width);
+  h = gst_video_format_get_component_height (state->format, 1, state->height);
   h = MIN (h, img->h >> img->y_chroma_shift);
-  w = gst_video_format_get_component_width (decoder->state.format, 1,
-      decoder->state.width);
+  w = gst_video_format_get_component_width (state->format, 1, state->width);
   w = MIN (w, img->w >> img->x_chroma_shift);
   for (i = 0; i < h; i++)
     memcpy (d + i * stride,
         img->planes[VPX_PLANE_U] + i * img->stride[VPX_PLANE_U], w);
 
   d = GST_BUFFER_DATA (buffer) +
-      gst_video_format_get_component_offset (decoder->state.format, 2,
-      decoder->state.width, decoder->state.height);
+      gst_video_format_get_component_offset (state->format, 2,
+      state->width, state->height);
   /* Same stride, height, width as above */
   for (i = 0; i < h; i++)
     memcpy (d + i * stride,
@@ -395,6 +387,7 @@ gst_vp8_dec_handle_frame (GstBaseVideoDecoder * decoder, GstVideoFrame * frame)
     int flags = 0;
     vpx_codec_stream_info_t stream_info;
     vpx_codec_caps_t caps;
+    GstVideoState *state = &GST_BASE_VIDEO_CODEC (dec)->state;
 
     memset (&stream_info, 0, sizeof (stream_info));
     stream_info.sz = sizeof (stream_info);
@@ -410,9 +403,9 @@ gst_vp8_dec_handle_frame (GstBaseVideoDecoder * decoder, GstVideoFrame * frame)
     }
 
     /* should set size here */
-    decoder->state.width = stream_info.w;
-    decoder->state.height = stream_info.h;
-    decoder->state.format = GST_VIDEO_FORMAT_I420;
+    state->width = stream_info.w;
+    state->height = stream_info.h;
+    state->format = GST_VIDEO_FORMAT_I420;
     gst_vp8_dec_send_tags (dec);
 
     caps = vpx_codec_get_caps (&vpx_codec_vp8_dx_algo);
diff --git a/ext/vp8/gstvp8enc.c b/ext/vp8/gstvp8enc.c
index 77a425e32a..e284191a29 100644
--- a/ext/vp8/gstvp8enc.c
+++ b/ext/vp8/gstvp8enc.c
@@ -672,34 +672,31 @@ static vpx_image_t *
 gst_vp8_enc_buffer_to_image (GstVP8Enc * enc, GstBuffer * buffer)
 {
   vpx_image_t *image = g_slice_new0 (vpx_image_t);
-  GstBaseVideoEncoder *encoder = (GstBaseVideoEncoder *) enc;
   guint8 *data = GST_BUFFER_DATA (buffer);
+  GstVideoState *state = &GST_BASE_VIDEO_CODEC (enc)->state;
 
   image->fmt = VPX_IMG_FMT_I420;
   image->bps = 12;
   image->x_chroma_shift = image->y_chroma_shift = 1;
   image->img_data = data;
-  image->w = image->d_w = encoder->state.width;
-  image->h = image->d_h = encoder->state.height;
+  image->w = image->d_w = state->width;
+  image->h = image->d_h = state->height;
 
   image->stride[VPX_PLANE_Y] =
-      gst_video_format_get_row_stride (encoder->state.format, 0,
-      encoder->state.width);
+      gst_video_format_get_row_stride (state->format, 0, state->width);
   image->stride[VPX_PLANE_U] =
-      gst_video_format_get_row_stride (encoder->state.format, 1,
-      encoder->state.width);
+      gst_video_format_get_row_stride (state->format, 1, state->width);
   image->stride[VPX_PLANE_V] =
-      gst_video_format_get_row_stride (encoder->state.format, 2,
-      encoder->state.width);
+      gst_video_format_get_row_stride (state->format, 2, state->width);
   image->planes[VPX_PLANE_Y] =
-      data + gst_video_format_get_component_offset (encoder->state.format, 0,
-      encoder->state.width, encoder->state.height);
+      data + gst_video_format_get_component_offset (state->format, 0,
+      state->width, state->height);
   image->planes[VPX_PLANE_U] =
-      data + gst_video_format_get_component_offset (encoder->state.format, 1,
-      encoder->state.width, encoder->state.height);
+      data + gst_video_format_get_component_offset (state->format, 1,
+      state->width, state->height);
   image->planes[VPX_PLANE_V] =
-      data + gst_video_format_get_component_offset (encoder->state.format, 2,
-      encoder->state.width, encoder->state.height);
+      data + gst_video_format_get_component_offset (state->format, 2,
+      state->width, state->height);
 
   return image;
 }
@@ -746,10 +743,10 @@ gst_vp8_enc_handle_frame (GstBaseVideoEncoder * base_video_encoder,
       return FALSE;
     }
 
-    cfg.g_w = base_video_encoder->state.width;
-    cfg.g_h = base_video_encoder->state.height;
-    cfg.g_timebase.num = base_video_encoder->state.fps_d;
-    cfg.g_timebase.den = base_video_encoder->state.fps_n;
+    cfg.g_w = state->width;
+    cfg.g_h = state->height;
+    cfg.g_timebase.num = state->fps_d;
+    cfg.g_timebase.den = state->fps_n;
 
     cfg.g_error_resilient = encoder->error_resilient;
     cfg.g_lag_in_frames = encoder->max_latency;
@@ -818,8 +815,7 @@ gst_vp8_enc_handle_frame (GstBaseVideoEncoder * base_video_encoder,
 
     gst_base_video_encoder_set_latency (base_video_encoder, 0,
         gst_util_uint64_scale (encoder->max_latency,
-            base_video_encoder->state.fps_d * GST_SECOND,
-            base_video_encoder->state.fps_n));
+            state->fps_d * GST_SECOND, state->fps_n));
     encoder->inited = TRUE;
   }
 
@@ -954,7 +950,8 @@ gst_vp8_enc_shape_output (GstBaseVideoEncoder * base_video_encoder,
     }
 
     GST_BUFFER_TIMESTAMP (buf) = gst_video_state_get_timestamp (state,
-        &base_video_encoder->segment, frame->presentation_frame_number);
+        &GST_BASE_VIDEO_CODEC (base_video_encoder)->segment,
+        frame->presentation_frame_number);
     GST_BUFFER_DURATION (buf) = 0;
     GST_BUFFER_OFFSET_END (buf) =
         _to_granulepos (frame->presentation_frame_number + 1,
@@ -963,7 +960,7 @@ gst_vp8_enc_shape_output (GstBaseVideoEncoder * base_video_encoder,
         gst_util_uint64_scale (frame->presentation_frame_number + 1,
         GST_SECOND * state->fps_d, state->fps_n);
 
-    gst_buffer_set_caps (buf, base_video_encoder->caps);
+    gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
     ret = gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
 
     if (ret != GST_FLOW_OK) {
@@ -984,18 +981,20 @@ gst_vp8_enc_shape_output (GstBaseVideoEncoder * base_video_encoder,
   }
 
   GST_BUFFER_TIMESTAMP (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment, frame->presentation_frame_number);
-  GST_BUFFER_DURATION (buf) = gst_video_state_get_timestamp (state,
-      &base_video_encoder->segment,
+      &GST_BASE_VIDEO_CODEC (base_video_encoder)->segment,
+      frame->presentation_frame_number);
+  GST_BUFFER_DURATION (buf) =
+      gst_video_state_get_timestamp (state,
+      &GST_BASE_VIDEO_CODEC (base_video_encoder)->segment,
       frame->presentation_frame_number + 1) - GST_BUFFER_TIMESTAMP (buf);
   GST_BUFFER_OFFSET_END (buf) =
-      _to_granulepos (frame->presentation_frame_number + 1,
-      0, encoder->keyframe_distance);
+      _to_granulepos (frame->presentation_frame_number + 1, 0,
+      encoder->keyframe_distance);
   GST_BUFFER_OFFSET (buf) =
       gst_util_uint64_scale (frame->presentation_frame_number + 1,
       GST_SECOND * state->fps_d, state->fps_n);
 
-  gst_buffer_set_caps (buf, base_video_encoder->caps);
+  gst_buffer_set_caps (buf, GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
 
   ret = gst_pad_push (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder), buf);
   if (ret != GST_FLOW_OK) {
diff --git a/gst-libs/gst/video/gstbasevideocodec.c b/gst-libs/gst/video/gstbasevideocodec.c
index f72a9b282c..39a13898a6 100644
--- a/gst-libs/gst/video/gstbasevideocodec.c
+++ b/gst-libs/gst/video/gstbasevideocodec.c
@@ -104,13 +104,27 @@ gst_base_video_codec_init (GstBaseVideoCodec * base_video_codec,
   gst_element_add_pad (GST_ELEMENT (base_video_codec),
       base_video_codec->srcpad);
 
+  gst_segment_init (&base_video_codec->segment, GST_FORMAT_TIME);
+
 }
 
 static void
 gst_base_video_codec_reset (GstBaseVideoCodec * base_video_codec)
 {
+  GList *g;
+
   GST_DEBUG ("reset");
 
+  for (g = base_video_codec->frames; g; g = g_list_next (g)) {
+    gst_base_video_codec_free_frame ((GstVideoFrame *) g->data);
+  }
+  g_list_free (base_video_codec->frames);
+
+  if (base_video_codec->caps) {
+    gst_caps_unref (base_video_codec->caps);
+    base_video_codec->caps = NULL;
+  }
+
 }
 
 static void
@@ -121,7 +135,6 @@ gst_base_video_codec_finalize (GObject * object)
   g_return_if_fail (GST_IS_BASE_VIDEO_CODEC (object));
   base_video_codec = GST_BASE_VIDEO_CODEC (object);
 
-
   G_OBJECT_CLASS (parent_class)->finalize (object);
 }
 
diff --git a/gst-libs/gst/video/gstbasevideocodec.h b/gst-libs/gst/video/gstbasevideocodec.h
index 9d24c5a8ba..450928670b 100644
--- a/gst-libs/gst/video/gstbasevideocodec.h
+++ b/gst-libs/gst/video/gstbasevideocodec.h
@@ -90,6 +90,18 @@ struct _GstBaseVideoCodec
   GstPad *srcpad;
 
   guint64 system_frame_number;
+
+  GList *frames;
+  GstVideoState state;
+  GstSegment segment;
+
+  GstCaps *caps;
+
+  gdouble proportion;
+  GstClockTime earliest_time;
+
+  /* FIXME before moving to base */
+  void *padding[GST_PADDING_LARGE];
 };
 
 struct _GstBaseVideoCodecClass
@@ -105,6 +117,8 @@ struct _GstBaseVideoCodecClass
   GstFlowReturn (*shape_output) (GstBaseVideoCodec *codec, GstVideoFrame *frame);
   GstCaps *(*get_caps) (GstBaseVideoCodec *codec);
 
+  /* FIXME before moving to base */
+  void *padding[GST_PADDING_LARGE];
 };
 
 GType gst_base_video_codec_get_type (void);
diff --git a/gst-libs/gst/video/gstbasevideodecoder.c b/gst-libs/gst/video/gstbasevideodecoder.c
index b60c64455f..bcad36717a 100644
--- a/gst-libs/gst/video/gstbasevideodecoder.c
+++ b/gst-libs/gst/video/gstbasevideodecoder.c
@@ -120,7 +120,6 @@ gst_base_video_decoder_init (GstBaseVideoDecoder * base_video_decoder,
   base_video_decoder->input_adapter = gst_adapter_new ();
   base_video_decoder->output_adapter = gst_adapter_new ();
 
-  gst_segment_init (&base_video_decoder->segment, GST_FORMAT_TIME);
   gst_base_video_decoder_reset (base_video_decoder);
 
   base_video_decoder->current_frame =
@@ -145,7 +144,7 @@ gst_base_video_decoder_sink_setcaps (GstPad * pad, GstCaps * caps)
 
   GST_DEBUG ("setcaps %" GST_PTR_FORMAT, caps);
 
-  state = &base_video_decoder->state;
+  state = &GST_BASE_VIDEO_CODEC (base_video_decoder)->state;
 
   if (state->codec_data) {
     gst_buffer_unref (state->codec_data);
@@ -242,7 +241,7 @@ gst_base_video_decoder_sink_event (GstPad * pad, GstEvent * event)
       gint64 start;
       gint64 stop;
       gint64 position;
-      GstSegment *segment = &base_video_decoder->segment;
+      GstSegment *segment = &GST_BASE_VIDEO_CODEC (base_video_decoder)->segment;
 
       gst_event_parse_new_segment_full (event, &update, &rate,
           &applied_rate, &format, &start, &stop, &position);
@@ -276,9 +275,11 @@ gst_base_video_decoder_sink_event (GstPad * pad, GstEvent * event)
       break;
     case GST_EVENT_FLUSH_STOP:{
       GST_OBJECT_LOCK (base_video_decoder);
-      base_video_decoder->earliest_time = GST_CLOCK_TIME_NONE;
-      base_video_decoder->proportion = 0.5;
-      gst_segment_init (&base_video_decoder->segment, GST_FORMAT_TIME);
+      GST_BASE_VIDEO_CODEC (base_video_decoder)->earliest_time =
+          GST_CLOCK_TIME_NONE;
+      GST_BASE_VIDEO_CODEC (base_video_decoder)->proportion = 0.5;
+      gst_segment_init (&GST_BASE_VIDEO_CODEC (base_video_decoder)->segment,
+          GST_FORMAT_TIME);
       GST_OBJECT_UNLOCK (base_video_decoder);
     }
     default:
@@ -357,22 +358,25 @@ gst_base_video_decoder_src_event (GstPad * pad, GstEvent * event)
       gst_event_parse_qos (event, &proportion, &diff, &timestamp);
 
       GST_OBJECT_LOCK (base_video_decoder);
-      base_video_decoder->proportion = proportion;
+      GST_BASE_VIDEO_CODEC (base_video_decoder)->proportion = proportion;
       if (G_LIKELY (GST_CLOCK_TIME_IS_VALID (timestamp))) {
         if (G_UNLIKELY (diff > 0)) {
-          if (base_video_decoder->state.fps_n > 0)
+          if (GST_BASE_VIDEO_CODEC (base_video_decoder)->state.fps_n > 0)
             duration =
                 gst_util_uint64_scale (GST_SECOND,
-                base_video_decoder->state.fps_d,
-                base_video_decoder->state.fps_n);
+                GST_BASE_VIDEO_CODEC (base_video_decoder)->state.fps_d,
+                GST_BASE_VIDEO_CODEC (base_video_decoder)->state.fps_n);
           else
             duration = 0;
-          base_video_decoder->earliest_time = timestamp + 2 * diff + duration;
+          GST_BASE_VIDEO_CODEC (base_video_decoder)->earliest_time =
+              timestamp + 2 * diff + duration;
         } else {
-          base_video_decoder->earliest_time = timestamp + diff;
+          GST_BASE_VIDEO_CODEC (base_video_decoder)->earliest_time =
+              timestamp + diff;
         }
       } else {
-        base_video_decoder->earliest_time = GST_CLOCK_TIME_NONE;
+        GST_BASE_VIDEO_CODEC (base_video_decoder)->earliest_time =
+            GST_CLOCK_TIME_NONE;
       }
       GST_OBJECT_UNLOCK (base_video_decoder);
 
@@ -547,7 +551,9 @@ gst_base_video_decoder_src_query (GstPad * pad, GstQuery * query)
       }
 
       time = enc->last_timestamp;
-      time = gst_segment_to_stream_time (&enc->segment, GST_FORMAT_TIME, time);
+      time =
+          gst_segment_to_stream_time (&GST_BASE_VIDEO_CODEC (enc)->segment,
+          GST_FORMAT_TIME, time);
 
       gst_query_set_position (query, format, time);
 
@@ -597,7 +603,8 @@ gst_base_video_decoder_sink_query (GstPad * pad, GstQuery * query)
   base_video_decoder = GST_BASE_VIDEO_DECODER (gst_pad_get_parent (pad));
 
   GST_DEBUG_OBJECT (base_video_decoder, "sink query fps=%d/%d",
-      base_video_decoder->state.fps_n, base_video_decoder->state.fps_d);
+      GST_BASE_VIDEO_CODEC (base_video_decoder)->state.fps_n,
+      GST_BASE_VIDEO_CODEC (base_video_decoder)->state.fps_d);
   switch (GST_QUERY_TYPE (query)) {
     case GST_QUERY_CONVERT:
     {
@@ -605,8 +612,9 @@ gst_base_video_decoder_sink_query (GstPad * pad, GstQuery * query)
       gint64 src_val, dest_val;
 
       gst_query_parse_convert (query, &src_fmt, &src_val, &dest_fmt, &dest_val);
-      res = gst_base_video_rawvideo_convert (&base_video_decoder->state,
-          src_fmt, src_val, &dest_fmt, &dest_val);
+      res =
+          gst_base_video_rawvideo_convert (&GST_BASE_VIDEO_CODEC
+          (base_video_decoder)->state, src_fmt, src_val, &dest_fmt, &dest_val);
       if (!res)
         goto error;
       gst_query_set_convert (query, src_fmt, src_val, dest_fmt, dest_val);
@@ -706,7 +714,6 @@ static void
 gst_base_video_decoder_reset (GstBaseVideoDecoder * base_video_decoder)
 {
   GstBaseVideoDecoderClass *base_video_decoder_class;
-  GList *g;
 
   base_video_decoder_class =
       GST_BASE_VIDEO_DECODER_GET_CLASS (base_video_decoder);
@@ -719,7 +726,7 @@ gst_base_video_decoder_reset (GstBaseVideoDecoder * base_video_decoder)
   base_video_decoder->have_sync = FALSE;
 
   base_video_decoder->timestamp_offset = GST_CLOCK_TIME_NONE;
-  base_video_decoder->system_frame_number = 0;
+  GST_BASE_VIDEO_CODEC (base_video_decoder)->system_frame_number = 0;
   base_video_decoder->presentation_frame_number = 0;
   base_video_decoder->base_picture_number = 0;
   base_video_decoder->last_timestamp = GST_CLOCK_TIME_NONE;
@@ -734,11 +741,6 @@ gst_base_video_decoder_reset (GstBaseVideoDecoder * base_video_decoder)
   if (base_video_decoder->output_adapter) {
     gst_adapter_clear (base_video_decoder->output_adapter);
   }
-
-  if (base_video_decoder->caps) {
-    gst_caps_unref (base_video_decoder->caps);
-    base_video_decoder->caps = NULL;
-  }
   //gst_segment_init (&base_video_decoder->segment, GST_FORMAT_TIME);
 
   if (base_video_decoder->current_frame) {
@@ -748,16 +750,10 @@ gst_base_video_decoder_reset (GstBaseVideoDecoder * base_video_decoder)
 
   base_video_decoder->have_src_caps = FALSE;
 
-  for (g = g_list_first (base_video_decoder->frames); g; g = g_list_next (g)) {
-    GstVideoFrame *frame = g->data;
-    gst_base_video_decoder_free_frame (frame);
-  }
-  g_list_free (base_video_decoder->frames);
-  base_video_decoder->frames = NULL;
-
   GST_OBJECT_LOCK (base_video_decoder);
-  base_video_decoder->earliest_time = GST_CLOCK_TIME_NONE;
-  base_video_decoder->proportion = 0.5;
+  GST_BASE_VIDEO_CODEC (base_video_decoder)->earliest_time =
+      GST_CLOCK_TIME_NONE;
+  GST_BASE_VIDEO_CODEC (base_video_decoder)->proportion = 0.5;
   GST_OBJECT_UNLOCK (base_video_decoder);
 
   if (base_video_decoder_class->reset) {
@@ -797,8 +793,9 @@ gst_base_video_decoder_chain (GstPad * pad, GstBuffer * buf)
     GST_WARNING
         ("Received buffer without a new-segment. Assuming timestamps start from 0.");
 
-    gst_segment_set_newsegment_full (&base_video_decoder->segment,
-        FALSE, 1.0, 1.0, GST_FORMAT_TIME, 0, GST_CLOCK_TIME_NONE, 0);
+    gst_segment_set_newsegment_full (&GST_BASE_VIDEO_CODEC
+        (base_video_decoder)->segment, FALSE, 1.0, 1.0, GST_FORMAT_TIME, 0,
+        GST_CLOCK_TIME_NONE, 0);
     base_video_decoder->have_segment = TRUE;
 
     event = gst_event_new_new_segment (FALSE, 1.0, GST_FORMAT_TIME, 0,
@@ -926,7 +923,8 @@ gst_base_video_decoder_change_state (GstElement * element,
       if (base_video_decoder_class->stop) {
         base_video_decoder_class->stop (base_video_decoder);
       }
-      gst_segment_init (&base_video_decoder->segment, GST_FORMAT_TIME);
+      gst_segment_init (&GST_BASE_VIDEO_CODEC (base_video_decoder)->segment,
+          GST_FORMAT_TIME);
       g_list_foreach (base_video_decoder->timestamps, (GFunc) g_free, NULL);
       g_list_free (base_video_decoder->timestamps);
       base_video_decoder->timestamps = NULL;
@@ -960,8 +958,9 @@ gst_base_video_decoder_new_frame (GstBaseVideoDecoder * base_video_decoder)
 
   frame = g_malloc0 (sizeof (GstVideoFrame));
 
-  frame->system_frame_number = base_video_decoder->system_frame_number;
-  base_video_decoder->system_frame_number++;
+  frame->system_frame_number =
+      GST_BASE_VIDEO_CODEC (base_video_decoder)->system_frame_number;
+  GST_BASE_VIDEO_CODEC (base_video_decoder)->system_frame_number++;
 
   frame->decode_frame_number = frame->system_frame_number -
       base_video_decoder->reorder_depth;
@@ -979,9 +978,14 @@ gst_base_video_decoder_finish_frame (GstBaseVideoDecoder * base_video_decoder,
     GstVideoFrame * frame)
 {
   GstBaseVideoDecoderClass *base_video_decoder_class;
+  GstVideoState *state = &GST_BASE_VIDEO_CODEC (base_video_decoder)->state;
   GstBuffer *src_buffer;
 
   GST_DEBUG ("finish frame");
+  GST_DEBUG ("n %d in %d out %d",
+      g_list_length (GST_BASE_VIDEO_CODEC (base_video_decoder)->frames),
+      gst_adapter_available (base_video_decoder->input_adapter),
+      gst_adapter_available (base_video_decoder->output_adapter));
 
   base_video_decoder_class =
       GST_BASE_VIDEO_DECODER_GET_CLASS (base_video_decoder);
@@ -994,7 +998,7 @@ gst_base_video_decoder_finish_frame (GstBaseVideoDecoder * base_video_decoder,
       GST_DEBUG ("sync timestamp %" GST_TIME_FORMAT " diff %" GST_TIME_FORMAT,
           GST_TIME_ARGS (frame->presentation_timestamp),
           GST_TIME_ARGS (frame->presentation_timestamp -
-              base_video_decoder->segment.start));
+              GST_BASE_VIDEO_CODEC (base_video_decoder)->segment.start));
       base_video_decoder->timestamp_offset = frame->presentation_timestamp;
       base_video_decoder->field_index &= 1;
     } else {
@@ -1010,7 +1014,7 @@ gst_base_video_decoder_finish_frame (GstBaseVideoDecoder * base_video_decoder,
         GST_WARNING
             ("No base timestamp.  Assuming frames start at segment start");
         base_video_decoder->timestamp_offset =
-            base_video_decoder->segment.start;
+            GST_BASE_VIDEO_CODEC (base_video_decoder)->segment.start;
         base_video_decoder->field_index &= 1;
       }
     }
@@ -1043,8 +1047,8 @@ gst_base_video_decoder_finish_frame (GstBaseVideoDecoder * base_video_decoder,
   base_video_decoder->last_timestamp = frame->presentation_timestamp;
 
   GST_BUFFER_FLAG_UNSET (frame->src_buffer, GST_BUFFER_FLAG_DELTA_UNIT);
-  if (base_video_decoder->state.interlaced) {
-    int tff = base_video_decoder->state.top_field_first;
+  if (state->interlaced) {
+    int tff = state->top_field_first;
 
     if (frame->field_index & 1) {
       tff ^= 1;
@@ -1075,8 +1079,8 @@ gst_base_video_decoder_finish_frame (GstBaseVideoDecoder * base_video_decoder,
   GST_DEBUG ("pushing frame %" GST_TIME_FORMAT,
       GST_TIME_ARGS (frame->presentation_timestamp));
 
-  base_video_decoder->frames =
-      g_list_remove (base_video_decoder->frames, frame);
+  GST_BASE_VIDEO_CODEC (base_video_decoder)->frames =
+      g_list_remove (GST_BASE_VIDEO_CODEC (base_video_decoder)->frames, frame);
 
   gst_base_video_decoder_set_src_caps (base_video_decoder);
 
@@ -1089,9 +1093,9 @@ gst_base_video_decoder_finish_frame (GstBaseVideoDecoder * base_video_decoder,
     gint64 start = GST_BUFFER_TIMESTAMP (src_buffer);
     gint64 stop = GST_BUFFER_TIMESTAMP (src_buffer) +
         GST_BUFFER_DURATION (src_buffer);
+    GstSegment *segment = &GST_BASE_VIDEO_CODEC (base_video_decoder)->segment;
 
-    if (gst_segment_clip (&base_video_decoder->segment, GST_FORMAT_TIME,
-            start, stop, &start, &stop)) {
+    if (gst_segment_clip (segment, GST_FORMAT_TIME, start, stop, &start, &stop)) {
       GST_BUFFER_TIMESTAMP (src_buffer) = start;
       GST_BUFFER_DURATION (src_buffer) = stop - start;
       GST_DEBUG ("accepting buffer inside segment: %" GST_TIME_FORMAT
@@ -1101,9 +1105,8 @@ gst_base_video_decoder_finish_frame (GstBaseVideoDecoder * base_video_decoder,
           GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (src_buffer)),
           GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (src_buffer) +
               GST_BUFFER_DURATION (src_buffer)),
-          GST_TIME_ARGS (base_video_decoder->segment.start),
-          GST_TIME_ARGS (base_video_decoder->segment.stop),
-          GST_TIME_ARGS (base_video_decoder->segment.time));
+          GST_TIME_ARGS (segment->start),
+          GST_TIME_ARGS (segment->stop), GST_TIME_ARGS (segment->time));
     } else {
       GST_DEBUG ("dropping buffer outside segment: %" GST_TIME_FORMAT
           " %" GST_TIME_FORMAT
@@ -1112,9 +1115,8 @@ gst_base_video_decoder_finish_frame (GstBaseVideoDecoder * base_video_decoder,
           GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (src_buffer)),
           GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (src_buffer) +
               GST_BUFFER_DURATION (src_buffer)),
-          GST_TIME_ARGS (base_video_decoder->segment.start),
-          GST_TIME_ARGS (base_video_decoder->segment.stop),
-          GST_TIME_ARGS (base_video_decoder->segment.time));
+          GST_TIME_ARGS (segment->start),
+          GST_TIME_ARGS (segment->stop), GST_TIME_ARGS (segment->time));
       gst_buffer_unref (src_buffer);
       return GST_FLOW_OK;
     }
@@ -1143,7 +1145,7 @@ gst_base_video_decoder_skip_frame (GstBaseVideoDecoder * base_video_decoder,
       GST_DEBUG ("sync timestamp %" GST_TIME_FORMAT " diff %" GST_TIME_FORMAT,
           GST_TIME_ARGS (frame->presentation_timestamp),
           GST_TIME_ARGS (frame->presentation_timestamp -
-              base_video_decoder->segment.start));
+              GST_BASE_VIDEO_CODEC (base_video_decoder)->segment.start));
       base_video_decoder->timestamp_offset = frame->presentation_timestamp;
       base_video_decoder->field_index = 0;
     } else {
@@ -1159,7 +1161,7 @@ gst_base_video_decoder_skip_frame (GstBaseVideoDecoder * base_video_decoder,
         GST_WARNING
             ("No base timestamp.  Assuming frames start at segment start");
         base_video_decoder->timestamp_offset =
-            base_video_decoder->segment.start;
+            GST_BASE_VIDEO_CODEC (base_video_decoder)->segment.start;
         base_video_decoder->field_index = 0;
       }
     }
@@ -1187,8 +1189,8 @@ gst_base_video_decoder_skip_frame (GstBaseVideoDecoder * base_video_decoder,
   GST_DEBUG ("skipping frame %" GST_TIME_FORMAT,
       GST_TIME_ARGS (frame->presentation_timestamp));
 
-  base_video_decoder->frames =
-      g_list_remove (base_video_decoder->frames, frame);
+  GST_BASE_VIDEO_CODEC (base_video_decoder)->frames =
+      g_list_remove (GST_BASE_VIDEO_CODEC (base_video_decoder)->frames, frame);
 
   gst_base_video_decoder_free_frame (frame);
 
@@ -1198,13 +1200,17 @@ gst_base_video_decoder_skip_frame (GstBaseVideoDecoder * base_video_decoder,
 int
 gst_base_video_decoder_get_height (GstBaseVideoDecoder * base_video_decoder)
 {
-  return base_video_decoder->state.height;
+  GstVideoState *state = &GST_BASE_VIDEO_CODEC (base_video_decoder)->state;
+
+  return state->height;
 }
 
 int
 gst_base_video_decoder_get_width (GstBaseVideoDecoder * base_video_decoder)
 {
-  return base_video_decoder->state.width;
+  GstVideoState *state = &GST_BASE_VIDEO_CODEC (base_video_decoder)->state;
+
+  return state->width;
 }
 
 GstFlowReturn
@@ -1212,7 +1218,7 @@ gst_base_video_decoder_end_of_stream (GstBaseVideoDecoder * base_video_decoder,
     GstBuffer * buffer)
 {
 
-  if (base_video_decoder->frames) {
+  if (GST_BASE_VIDEO_CODEC (base_video_decoder)->frames) {
     GST_DEBUG ("EOS with frames left over");
   }
 
@@ -1244,20 +1250,20 @@ static guint64
 gst_base_video_decoder_get_timestamp (GstBaseVideoDecoder * base_video_decoder,
     int picture_number)
 {
-  if (base_video_decoder->state.fps_d == 0) {
+  GstVideoState *state = &GST_BASE_VIDEO_CODEC (base_video_decoder)->state;
+
+  if (state->fps_d == 0) {
     return -1;
   }
   if (picture_number < base_video_decoder->base_picture_number) {
     return base_video_decoder->timestamp_offset -
         (gint64) gst_util_uint64_scale (base_video_decoder->base_picture_number
-        - picture_number, base_video_decoder->state.fps_d * GST_SECOND,
-        base_video_decoder->state.fps_n);
+        - picture_number, state->fps_d * GST_SECOND, state->fps_n);
   } else {
     return base_video_decoder->timestamp_offset +
         gst_util_uint64_scale (picture_number -
         base_video_decoder->base_picture_number,
-        base_video_decoder->state.fps_d * GST_SECOND,
-        base_video_decoder->state.fps_n);
+        state->fps_d * GST_SECOND, state->fps_n);
   }
 }
 
@@ -1265,7 +1271,9 @@ static guint64
 gst_base_video_decoder_get_field_timestamp (GstBaseVideoDecoder *
     base_video_decoder, int field_offset)
 {
-  if (base_video_decoder->state.fps_d == 0) {
+  GstVideoState *state = &GST_BASE_VIDEO_CODEC (base_video_decoder)->state;
+
+  if (state->fps_d == 0) {
     return GST_CLOCK_TIME_NONE;
   }
   if (field_offset < 0) {
@@ -1273,25 +1281,25 @@ gst_base_video_decoder_get_field_timestamp (GstBaseVideoDecoder *
     return GST_CLOCK_TIME_NONE;
   }
   return base_video_decoder->timestamp_offset +
-      gst_util_uint64_scale (field_offset,
-      base_video_decoder->state.fps_d * GST_SECOND,
-      base_video_decoder->state.fps_n * 2);
+      gst_util_uint64_scale (field_offset, state->fps_d * GST_SECOND,
+      state->fps_n * 2);
 }
 
 static guint64
 gst_base_video_decoder_get_field_duration (GstBaseVideoDecoder *
     base_video_decoder, int n_fields)
 {
-  if (base_video_decoder->state.fps_d == 0) {
+  GstVideoState *state = &GST_BASE_VIDEO_CODEC (base_video_decoder)->state;
+
+  if (state->fps_d == 0) {
     return GST_CLOCK_TIME_NONE;
   }
   if (n_fields < 0) {
     GST_WARNING ("n_fields < 0");
     return GST_CLOCK_TIME_NONE;
   }
-  return gst_util_uint64_scale (n_fields,
-      base_video_decoder->state.fps_d * GST_SECOND,
-      base_video_decoder->state.fps_n * 2);
+  return gst_util_uint64_scale (n_fields, state->fps_d * GST_SECOND,
+      state->fps_n * 2);
 }
 
 
@@ -1345,11 +1353,13 @@ gst_base_video_decoder_have_frame_2 (GstBaseVideoDecoder * base_video_decoder)
   GST_DEBUG ("dts %" GST_TIME_FORMAT, GST_TIME_ARGS (frame->decode_timestamp));
   GST_DEBUG ("dist %d", frame->distance_from_sync);
 
-  base_video_decoder->frames = g_list_append (base_video_decoder->frames,
-      frame);
+  GST_BASE_VIDEO_CODEC (base_video_decoder)->frames =
+      g_list_append (GST_BASE_VIDEO_CODEC (base_video_decoder)->frames, frame);
 
-  frame->deadline = gst_segment_to_running_time (&base_video_decoder->segment,
-      GST_FORMAT_TIME, frame->presentation_timestamp);
+  frame->deadline =
+      gst_segment_to_running_time (&GST_BASE_VIDEO_CODEC
+      (base_video_decoder)->segment, GST_FORMAT_TIME,
+      frame->presentation_timestamp);
 
   /* do something with frame */
   ret = base_video_decoder_class->handle_frame (base_video_decoder, frame);
@@ -1367,7 +1377,7 @@ gst_base_video_decoder_have_frame_2 (GstBaseVideoDecoder * base_video_decoder)
 GstVideoState *
 gst_base_video_decoder_get_state (GstBaseVideoDecoder * base_video_decoder)
 {
-  return &base_video_decoder->state;
+  return &GST_BASE_VIDEO_CODEC (base_video_decoder)->state;
 
 }
 
@@ -1375,7 +1385,8 @@ void
 gst_base_video_decoder_set_state (GstBaseVideoDecoder * base_video_decoder,
     GstVideoState * state)
 {
-  memcpy (&base_video_decoder->state, state, sizeof (*state));
+  memcpy (&GST_BASE_VIDEO_CODEC (base_video_decoder)->state,
+      state, sizeof (*state));
 
 }
 
@@ -1408,7 +1419,7 @@ gst_base_video_decoder_get_oldest_frame (GstBaseVideoDecoder *
 {
   GList *g;
 
-  g = g_list_first (base_video_decoder->frames);
+  g = g_list_first (GST_BASE_VIDEO_CODEC (base_video_decoder)->frames);
 
   if (g == NULL)
     return NULL;
@@ -1421,7 +1432,8 @@ gst_base_video_decoder_get_frame (GstBaseVideoDecoder * base_video_decoder,
 {
   GList *g;
 
-  for (g = g_list_first (base_video_decoder->frames); g; g = g_list_next (g)) {
+  for (g = g_list_first (GST_BASE_VIDEO_CODEC (base_video_decoder)->frames);
+      g; g = g_list_next (g)) {
     GstVideoFrame *frame = g->data;
 
     if (frame->system_frame_number == frame_number) {
@@ -1436,7 +1448,7 @@ void
 gst_base_video_decoder_set_src_caps (GstBaseVideoDecoder * base_video_decoder)
 {
   GstCaps *caps;
-  GstVideoState *state = &base_video_decoder->state;
+  GstVideoState *state = &GST_BASE_VIDEO_CODEC (base_video_decoder)->state;
 
   if (base_video_decoder->have_src_caps)
     return;
@@ -1462,11 +1474,12 @@ gst_base_video_decoder_alloc_src_frame (GstBaseVideoDecoder *
 {
   GstFlowReturn flow_ret;
   int num_bytes;
+  GstVideoState *state = &GST_BASE_VIDEO_CODEC (base_video_decoder)->state;
 
   gst_base_video_decoder_set_src_caps (base_video_decoder);
 
-  num_bytes = gst_video_format_get_size (base_video_decoder->state.format,
-      base_video_decoder->state.width, base_video_decoder->state.height);
+  num_bytes = gst_video_format_get_size (state->format, state->width,
+      state->height);
   flow_ret =
       gst_pad_alloc_buffer_and_set_caps (GST_BASE_VIDEO_CODEC_SRC_PAD
       (base_video_decoder), GST_BUFFER_OFFSET_NONE, num_bytes,
@@ -1485,10 +1498,11 @@ gst_base_video_decoder_get_max_decode_time (GstBaseVideoDecoder *
     base_video_decoder, GstVideoFrame * frame)
 {
   GstClockTimeDiff deadline;
+  GstClockTime earliest_time;
 
-  if (GST_CLOCK_TIME_IS_VALID (base_video_decoder->earliest_time))
-    deadline = GST_CLOCK_DIFF (base_video_decoder->earliest_time,
-        frame->deadline);
+  earliest_time = GST_BASE_VIDEO_CODEC (base_video_decoder)->earliest_time;
+  if (GST_CLOCK_TIME_IS_VALID (earliest_time))
+    deadline = GST_CLOCK_DIFF (earliest_time, frame->deadline);
   else
     deadline = G_MAXINT64;
 
diff --git a/gst-libs/gst/video/gstbasevideodecoder.h b/gst-libs/gst/video/gstbasevideodecoder.h
index f996a587ec..f193fa5252 100644
--- a/gst-libs/gst/video/gstbasevideodecoder.h
+++ b/gst-libs/gst/video/gstbasevideodecoder.h
@@ -73,21 +73,14 @@ struct _GstBaseVideoDecoder
   GstAdapter *input_adapter;
   GstAdapter *output_adapter;
 
-  GList *frames;
-
   gboolean have_sync;
   gboolean discont;
   gboolean started;
 
-  GstVideoState state;
-  GstSegment segment;
-
   gboolean sink_clipping;
 
   guint64 presentation_frame_number;
-  guint64 system_frame_number;
 
-  GstCaps *caps;
   gboolean have_src_caps;
 
   GstVideoFrame *current_frame;
@@ -99,9 +92,6 @@ struct _GstBaseVideoDecoder
 
   GstClockTime timestamp_offset;
 
-  gdouble proportion;
-  GstClockTime earliest_time;
-
   //GstBuffer *codec_data;
 
   guint64 input_offset;
@@ -117,6 +107,9 @@ struct _GstBaseVideoDecoder
 
   GList *timestamps;
   gboolean have_segment;
+
+  /* FIXME before moving to base */
+  void *padding[GST_PADDING_LARGE];
 };
 
 struct _GstBaseVideoDecoderClass
@@ -139,6 +132,9 @@ struct _GstBaseVideoDecoderClass
 
   guint32 capture_mask;
   guint32 capture_pattern;
+
+  /* FIXME before moving to base */
+  void *padding[GST_PADDING_LARGE];
 };
 
 GType gst_base_video_decoder_get_type (void);
diff --git a/gst-libs/gst/video/gstbasevideoencoder.c b/gst-libs/gst/video/gstbasevideoencoder.c
index 9a1da19ec2..fb518fc1b2 100644
--- a/gst-libs/gst/video/gstbasevideoencoder.c
+++ b/gst-libs/gst/video/gstbasevideoencoder.c
@@ -110,7 +110,7 @@ gst_base_video_encoder_sink_setcaps (GstPad * pad, GstCaps * caps)
 
   GST_DEBUG ("setcaps");
 
-  state = &base_video_encoder->state;
+  state = &GST_BASE_VIDEO_CODEC (base_video_encoder)->state;
   structure = gst_caps_get_structure (caps, 0);
 
   gst_video_format_parse_caps (caps, &state->format,
@@ -137,7 +137,7 @@ gst_base_video_encoder_sink_setcaps (GstPad * pad, GstCaps * caps)
   state->clean_offset_top = 0;
 
   ret = base_video_encoder_class->set_format (base_video_encoder,
-      &base_video_encoder->state);
+      &GST_BASE_VIDEO_CODEC (base_video_encoder)->state);
   if (ret) {
     ret = base_video_encoder_class->start (base_video_encoder);
   }
@@ -152,7 +152,6 @@ gst_base_video_encoder_finalize (GObject * object)
 {
   GstBaseVideoEncoder *base_video_encoder;
   GstBaseVideoEncoderClass *base_video_encoder_class;
-  GList *g;
 
   g_return_if_fail (GST_IS_BASE_VIDEO_ENCODER (object));
   base_video_encoder = GST_BASE_VIDEO_ENCODER (object);
@@ -160,16 +159,6 @@ gst_base_video_encoder_finalize (GObject * object)
 
   GST_DEBUG ("finalize");
 
-  for (g = base_video_encoder->frames; g; g = g_list_next (g)) {
-    gst_base_video_codec_free_frame ((GstVideoFrame *) g->data);
-  }
-  g_list_free (base_video_encoder->frames);
-
-  if (base_video_encoder->caps) {
-    gst_caps_unref (base_video_encoder->caps);
-    base_video_encoder->caps = NULL;
-  }
-
   G_OBJECT_CLASS (parent_class)->finalize (object);
 }
 
@@ -215,8 +204,9 @@ gst_base_video_encoder_sink_event (GstPad * pad, GstEvent * event)
       GST_DEBUG ("new segment %" GST_TIME_FORMAT " %" GST_TIME_FORMAT,
           GST_TIME_ARGS (start), GST_TIME_ARGS (position));
 
-      gst_segment_set_newsegment_full (&base_video_encoder->segment,
-          update, rate, applied_rate, format, start, stop, position);
+      gst_segment_set_newsegment_full (&GST_BASE_VIDEO_CODEC
+          (base_video_encoder)->segment, update, rate, applied_rate, format,
+          start, stop, position);
 
       ret =
           gst_pad_push_event (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder),
@@ -338,8 +328,8 @@ gst_base_video_encoder_src_query (GstPad * pad, GstQuery * query)
 
       gst_query_parse_convert (query, &src_fmt, &src_val, &dest_fmt, &dest_val);
       res =
-          gst_base_video_encoded_video_convert (&enc->state, src_fmt, src_val,
-          &dest_fmt, &dest_val);
+          gst_base_video_encoded_video_convert (&GST_BASE_VIDEO_CODEC
+          (enc)->state, src_fmt, src_val, &dest_fmt, &dest_val);
       if (!res)
         goto error;
       gst_query_set_convert (query, src_fmt, src_val, dest_fmt, dest_val);
@@ -413,7 +403,7 @@ gst_base_video_encoder_chain (GstPad * pad, GstBuffer * buf)
     gint64 clip_start;
     gint64 clip_stop;
 
-    if (!gst_segment_clip (&base_video_encoder->segment,
+    if (!gst_segment_clip (&GST_BASE_VIDEO_CODEC (base_video_encoder)->segment,
             GST_FORMAT_TIME, start, stop, &clip_start, &clip_stop)) {
       GST_DEBUG ("clipping to segment dropped frame");
       goto done;
@@ -430,8 +420,8 @@ gst_base_video_encoder_chain (GstPad * pad, GstBuffer * buf)
       base_video_encoder->presentation_frame_number;
   base_video_encoder->presentation_frame_number++;
 
-  base_video_encoder->frames =
-      g_list_append (base_video_encoder->frames, frame);
+  GST_BASE_VIDEO_CODEC (base_video_encoder)->frames =
+      g_list_append (GST_BASE_VIDEO_CODEC (base_video_encoder)->frames, frame);
 
   klass->handle_frame (base_video_encoder, frame);
 
@@ -482,8 +472,9 @@ gst_base_video_encoder_finish_frame (GstBaseVideoEncoder * base_video_encoder,
   base_video_encoder_class =
       GST_BASE_VIDEO_ENCODER_GET_CLASS (base_video_encoder);
 
-  frame->system_frame_number = base_video_encoder->system_frame_number;
-  base_video_encoder->system_frame_number++;
+  frame->system_frame_number =
+      GST_BASE_VIDEO_CODEC (base_video_encoder)->system_frame_number;
+  GST_BASE_VIDEO_CODEC (base_video_encoder)->system_frame_number++;
 
   if (frame->is_sync_point) {
     base_video_encoder->distance_from_sync = 0;
@@ -500,41 +491,46 @@ gst_base_video_encoder_finish_frame (GstBaseVideoEncoder * base_video_encoder,
     frame->decode_timestamp = 0;
   } else {
     frame->decode_timestamp = gst_util_uint64_scale (frame->decode_frame_number,
-        GST_SECOND * base_video_encoder->state.fps_d,
-        base_video_encoder->state.fps_n);
+        GST_SECOND * GST_BASE_VIDEO_CODEC (base_video_encoder)->state.fps_d,
+        GST_BASE_VIDEO_CODEC (base_video_encoder)->state.fps_n);
   }
 
   GST_BUFFER_TIMESTAMP (frame->src_buffer) = frame->presentation_timestamp;
   GST_BUFFER_DURATION (frame->src_buffer) = frame->presentation_duration;
   GST_BUFFER_OFFSET (frame->src_buffer) = frame->decode_timestamp;
 
-  base_video_encoder->frames =
-      g_list_remove (base_video_encoder->frames, frame);
+  GST_BASE_VIDEO_CODEC (base_video_encoder)->frames =
+      g_list_remove (GST_BASE_VIDEO_CODEC (base_video_encoder)->frames, frame);
 
   if (!base_video_encoder->set_output_caps) {
     if (base_video_encoder_class->get_caps) {
-      base_video_encoder->caps =
+      GST_BASE_VIDEO_CODEC (base_video_encoder)->caps =
           base_video_encoder_class->get_caps (base_video_encoder);
     } else {
-      base_video_encoder->caps = gst_caps_new_simple ("video/unknown", NULL);
+      GST_BASE_VIDEO_CODEC (base_video_encoder)->caps =
+          gst_caps_new_simple ("video/unknown", NULL);
     }
     gst_pad_set_caps (GST_BASE_VIDEO_CODEC_SRC_PAD (base_video_encoder),
-        base_video_encoder->caps);
+        GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
     base_video_encoder->set_output_caps = TRUE;
   }
 
   gst_buffer_set_caps (GST_BUFFER (frame->src_buffer),
-      base_video_encoder->caps);
+      GST_BASE_VIDEO_CODEC (base_video_encoder)->caps);
 
   if (frame->force_keyframe) {
     GstClockTime stream_time;
     GstClockTime running_time;
     GstStructure *s;
 
-    running_time = gst_segment_to_running_time (&base_video_encoder->segment,
-        GST_FORMAT_TIME, frame->presentation_timestamp);
-    stream_time = gst_segment_to_stream_time (&base_video_encoder->segment,
-        GST_FORMAT_TIME, frame->presentation_timestamp);
+    running_time =
+        gst_segment_to_running_time (&GST_BASE_VIDEO_CODEC
+        (base_video_encoder)->segment, GST_FORMAT_TIME,
+        frame->presentation_timestamp);
+    stream_time =
+        gst_segment_to_stream_time (&GST_BASE_VIDEO_CODEC
+        (base_video_encoder)->segment, GST_FORMAT_TIME,
+        frame->presentation_timestamp);
 
     /* FIXME this should send the event that we got on the sink pad
        instead of creating a new one */
@@ -563,19 +559,19 @@ gst_base_video_encoder_finish_frame (GstBaseVideoEncoder * base_video_encoder,
 int
 gst_base_video_encoder_get_height (GstBaseVideoEncoder * base_video_encoder)
 {
-  return base_video_encoder->state.height;
+  return GST_BASE_VIDEO_CODEC (base_video_encoder)->state.height;
 }
 
 int
 gst_base_video_encoder_get_width (GstBaseVideoEncoder * base_video_encoder)
 {
-  return base_video_encoder->state.width;
+  return GST_BASE_VIDEO_CODEC (base_video_encoder)->state.width;
 }
 
 const GstVideoState *
 gst_base_video_encoder_get_state (GstBaseVideoEncoder * base_video_encoder)
 {
-  return &base_video_encoder->state;
+  return &GST_BASE_VIDEO_CODEC (base_video_encoder)->state;
 }
 
 GstFlowReturn
@@ -583,7 +579,7 @@ gst_base_video_encoder_end_of_stream (GstBaseVideoEncoder * base_video_encoder,
     GstBuffer * buffer)
 {
 
-  if (base_video_encoder->frames) {
+  if (GST_BASE_VIDEO_CODEC (base_video_encoder)->frames) {
     GST_WARNING ("EOS with frames left over");
   }
 
@@ -612,8 +608,8 @@ gst_base_video_encoder_set_latency_fields (GstBaseVideoEncoder *
   gint64 latency;
 
   latency = gst_util_uint64_scale (n_fields,
-      base_video_encoder->state.fps_d * GST_SECOND,
-      2 * base_video_encoder->state.fps_n);
+      GST_BASE_VIDEO_CODEC (base_video_encoder)->state.fps_d * GST_SECOND,
+      2 * GST_BASE_VIDEO_CODEC (base_video_encoder)->state.fps_n);
 
   gst_base_video_encoder_set_latency (base_video_encoder, latency, latency);
 
@@ -625,7 +621,7 @@ gst_base_video_encoder_get_oldest_frame (GstBaseVideoEncoder *
 {
   GList *g;
 
-  g = g_list_first (base_video_encoder->frames);
+  g = g_list_first (GST_BASE_VIDEO_CODEC (base_video_encoder)->frames);
 
   if (g == NULL)
     return NULL;
diff --git a/gst-libs/gst/video/gstbasevideoencoder.h b/gst-libs/gst/video/gstbasevideoencoder.h
index 9c64a363db..759519630c 100644
--- a/gst-libs/gst/video/gstbasevideoencoder.h
+++ b/gst-libs/gst/video/gstbasevideoencoder.h
@@ -65,24 +65,20 @@ struct _GstBaseVideoEncoder
   GstBaseVideoCodec base_video_codec;
 
   /*< private >*/
-  GList *frames;
-
-  GstVideoState state;
-  GstSegment segment;
-
   gboolean sink_clipping;
 
   guint64 presentation_frame_number;
-  guint64 system_frame_number;
   int distance_from_sync;
 
-  GstCaps *caps;
   gboolean set_output_caps;
 
   gint64 min_latency;
   gint64 max_latency;
 
   gboolean force_keyframe;
+
+  /* FIXME before moving to base */
+  void *padding[GST_PADDING_LARGE];
 };
 
 struct _GstBaseVideoEncoderClass
@@ -97,6 +93,8 @@ struct _GstBaseVideoEncoderClass
   GstFlowReturn (*shape_output) (GstBaseVideoEncoder *coder, GstVideoFrame *frame);
   GstCaps *(*get_caps) (GstBaseVideoEncoder *coder);
 
+  /* FIXME before moving to base */
+  void *padding[GST_PADDING_LARGE];
 };
 
 GType gst_base_video_encoder_get_type (void);