diff --git a/gst/matroska/matroska-mux.c b/gst/matroska/matroska-mux.c
index 93219bafe8..7ae340a6fc 100644
--- a/gst/matroska/matroska-mux.c
+++ b/gst/matroska/matroska-mux.c
@@ -498,6 +498,7 @@ gst_matroska_mux_init (GstMatroskaMux * mux, gpointer g_class)
   mux->num_a_streams = 0;
   mux->num_t_streams = 0;
   mux->num_v_streams = 0;
+  mux->internal_toc = NULL;
 
   /* create used uid list */
   mux->used_uids = g_array_sized_new (FALSE, FALSE, sizeof (guint64), 10);
@@ -526,8 +527,9 @@ gst_matroska_mux_finalize (GObject * object)
 
   g_array_free (mux->used_uids, TRUE);
 
-  if (mux->internal_toc != NULL) {
+  if (mux->internal_toc) {
     gst_toc_unref (mux->internal_toc);
+    mux->internal_toc = NULL;
   }
 
   G_OBJECT_CLASS (parent_class)->finalize (object);
@@ -703,7 +705,10 @@ gst_matroska_mux_reset (GstElement * element)
 
   /* reset chapters */
   gst_toc_setter_reset (GST_TOC_SETTER (mux));
-  mux->internal_toc = NULL;
+  if (mux->internal_toc) {
+    gst_toc_unref (mux->internal_toc);
+    mux->internal_toc = NULL;
+  }
 
   mux->chapters_pos = 0;
 
@@ -3230,6 +3235,7 @@ gst_matroska_mux_finish (GstMatroskaMux * mux)
   guint64 duration = 0;
   GSList *collected;
   const GstTagList *tags, *toc_tags;
+  const GstToc *toc;
   gboolean has_main_tags, toc_has_tags = FALSE;
   GList *cur;
 
@@ -3270,9 +3276,9 @@ gst_matroska_mux_finish (GstMatroskaMux * mux)
   /* tags */
   tags = gst_tag_setter_get_tag_list (GST_TAG_SETTER (mux));
   has_main_tags = tags != NULL && !gst_matroska_mux_tag_list_is_empty (tags);
+  toc = gst_toc_setter_get_toc (GST_TOC_SETTER (mux));
 
-  if (has_main_tags || gst_matroska_mux_streams_have_tags (mux)
-      || gst_toc_setter_get_toc (GST_TOC_SETTER (mux)) != NULL) {
+  if (has_main_tags || gst_matroska_mux_streams_have_tags (mux) || toc != NULL) {
     guint64 master_tags = 0, master_tag;
 
     GST_DEBUG_OBJECT (mux, "Writing tags");
@@ -3327,8 +3333,7 @@ gst_matroska_mux_finish (GstMatroskaMux * mux)
       mux->info_pos - mux->segment_master);
   gst_ebml_replace_uint (ebml, mux->seekhead_pos + 60,
       mux->tracks_pos - mux->segment_master);
-  if (gst_toc_setter_get_toc (GST_TOC_SETTER (mux)) != NULL
-      && mux->chapters_pos > 0) {
+  if (toc != NULL && mux->chapters_pos > 0) {
     gst_ebml_replace_uint (ebml, mux->seekhead_pos + 88,
         mux->chapters_pos - mux->segment_master);
   } else {
@@ -3363,6 +3368,10 @@ gst_matroska_mux_finish (GstMatroskaMux * mux)
     gst_ebml_write_seek (ebml, my_pos);
   }
 
+  if (toc != NULL) {
+    gst_toc_unref (toc);
+  }
+
   /* loop tracks:
    * - first get the overall duration
    *   (a released track may have left a duration in here)
diff --git a/gst/matroska/matroska-read-common.c b/gst/matroska/matroska-read-common.c
index fcce782c78..419ba7f95f 100644
--- a/gst/matroska/matroska-read-common.c
+++ b/gst/matroska/matroska-read-common.c
@@ -2897,6 +2897,9 @@ gst_matroska_read_common_init (GstMatroskaReadCommon * ctx)
   ctx->index = NULL;
   ctx->global_tags = NULL;
   ctx->adapter = gst_adapter_new ();
+  ctx->toc = NULL;
+  ctx->internal_toc = NULL;
+  ctx->toc_updated = FALSE;
   ctx->cached_track_taglists =
       g_hash_table_new_full (NULL, NULL, NULL,
       (GDestroyNotify) gst_tag_list_unref);
@@ -2915,6 +2918,17 @@ gst_matroska_read_common_finalize (GstMatroskaReadCommon * ctx)
     ctx->global_tags = NULL;
   }
 
+  if (ctx->toc) {
+    gst_toc_unref (ctx->toc);
+    ctx->toc = NULL;
+  }
+  if (ctx->internal_toc) {
+    gst_toc_unref (ctx->internal_toc);
+    ctx->internal_toc = NULL;
+  }
+
+  ctx->toc_updated = FALSE;
+
   g_object_unref (ctx->adapter);
   g_hash_table_remove_all (ctx->cached_track_taglists);
   g_hash_table_unref (ctx->cached_track_taglists);
@@ -3010,6 +3024,7 @@ gst_matroska_read_common_reset (GstElement * element,
     gst_toc_unref (ctx->internal_toc);
     ctx->internal_toc = NULL;
   }
+  ctx->toc_updated = FALSE;
 }
 
 /* call with object lock held */
diff --git a/tests/check/elements/matroskademux.c b/tests/check/elements/matroskademux.c
index 60b4c69112..5081c8494a 100644
--- a/tests/check/elements/matroskademux.c
+++ b/tests/check/elements/matroskademux.c
@@ -118,7 +118,7 @@ GST_START_TEST (test_sub_terminator)
   buf = gst_buffer_new_wrapped (mkv_data, mkv_size);
   GST_BUFFER_OFFSET (buf) = 0;
 
-  fail_unless_equals_int (gst_harness_push (h, buf), GST_FLOW_OK);
+  fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf));
   gst_harness_push_event (h, gst_event_new_eos ());
 
   pull_and_check_buffer (h, 1 * GST_SECOND, 2 * GST_SECOND, "foo");
@@ -271,7 +271,7 @@ GST_START_TEST (test_toc_demux)
   buf = gst_buffer_new_wrapped (mkv_data, mkv_size);
   GST_BUFFER_OFFSET (buf) = 0;
 
-  fail_unless_equals_int (gst_harness_push (h, buf), GST_FLOW_OK);
+  fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf));
   gst_harness_push_event (h, gst_event_new_eos ());
 
   event = gst_harness_try_pull_event (h);
@@ -280,8 +280,10 @@ GST_START_TEST (test_toc_demux)
   while (event != NULL) {
     if (event->type == GST_EVENT_TOC) {
       gst_event_parse_toc (event, &demuxed_toc, &update);
+      gst_event_unref (event);
       break;
     }
+    gst_event_unref (event);
     event = gst_harness_try_pull_event (h);
   }
 
diff --git a/tests/check/elements/matroskamux.c b/tests/check/elements/matroskamux.c
index 8f3a053cb8..bf804bbc79 100644
--- a/tests/check/elements/matroskamux.c
+++ b/tests/check/elements/matroskamux.c
@@ -939,6 +939,7 @@ test_toc (gboolean with_edition)
 
   /* send eos to ensure everything is written */
   fail_unless (gst_harness_push_event (h, gst_event_new_eos ()));
+  ASSERT_MINI_OBJECT_REFCOUNT (test_toc, "test_toc", 1);
 
   outbuffer = gst_harness_pull (h);
   fail_unless (outbuffer != NULL);