diff --git a/omx/gstomxvideodec.c b/omx/gstomxvideodec.c
index 9e7da441f6..487652224f 100644
--- a/omx/gstomxvideodec.c
+++ b/omx/gstomxvideodec.c
@@ -793,6 +793,7 @@ component_error:
         gst_event_new_eos ());
     gst_pad_pause_task (GST_BASE_VIDEO_CODEC_SRC_PAD (self));
     self->downstream_flow_ret = GST_FLOW_ERROR;
+    self->started = FALSE;
     return;
   }
 
@@ -801,6 +802,7 @@ flushing:
     GST_DEBUG_OBJECT (self, "Flushing -- stopping task");
     gst_pad_pause_task (GST_BASE_VIDEO_CODEC_SRC_PAD (self));
     self->downstream_flow_ret = GST_FLOW_WRONG_STATE;
+    self->started = FALSE;
     return;
   }
 
@@ -821,6 +823,7 @@ flow_error:
           gst_event_new_eos ());
       gst_pad_pause_task (GST_BASE_VIDEO_CODEC_SRC_PAD (self));
     }
+    self->started = FALSE;
     return;
   }
 
@@ -832,6 +835,7 @@ reconfigure_error:
         gst_event_new_eos ());
     gst_pad_pause_task (GST_BASE_VIDEO_CODEC_SRC_PAD (self));
     self->downstream_flow_ret = GST_FLOW_ERROR;
+    self->started = FALSE;
     return;
   }
 
@@ -843,6 +847,7 @@ invalid_buffer:
         gst_event_new_eos ());
     gst_pad_pause_task (GST_BASE_VIDEO_CODEC_SRC_PAD (self));
     self->downstream_flow_ret = GST_FLOW_NOT_NEGOTIATED;
+    self->started = FALSE;
     return;
   }
 
@@ -853,6 +858,7 @@ caps_failed:
         gst_event_new_eos ());
     gst_pad_pause_task (GST_BASE_VIDEO_CODEC_SRC_PAD (self));
     self->downstream_flow_ret = GST_FLOW_NOT_NEGOTIATED;
+    self->started = FALSE;
     return;
   }
 }
@@ -865,6 +871,7 @@ gst_omx_video_dec_start (GstBaseVideoDecoder * decoder)
 
   self = GST_OMX_VIDEO_DEC (decoder);
 
+  self->eos = FALSE;
   self->downstream_flow_ret = GST_FLOW_OK;
   ret =
       gst_pad_start_task (GST_BASE_VIDEO_CODEC_SRC_PAD (self),
@@ -888,6 +895,8 @@ gst_omx_video_dec_stop (GstBaseVideoDecoder * decoder)
     gst_omx_component_set_state (self->component, OMX_StateIdle);
 
   self->downstream_flow_ret = GST_FLOW_WRONG_STATE;
+  self->started = FALSE;
+  self->eos = FALSE;
 
   g_mutex_lock (self->drain_lock);
   self->draining = FALSE;
@@ -1068,10 +1077,7 @@ gst_omx_video_dec_set_format (GstBaseVideoDecoder * decoder,
   }
 
   if (needs_disable && is_format_change) {
-    if (self->started) {
-      gst_omx_video_dec_drain (self);
-      self->started = FALSE;
-    }
+    gst_omx_video_dec_drain (self);
 
     /* FIXME: Workaround for 
      * https://bugzilla.gnome.org/show_bug.cgi?id=654529
@@ -1182,8 +1188,7 @@ gst_omx_video_dec_reset (GstBaseVideoDecoder * decoder)
 
   GST_DEBUG_OBJECT (self, "Resetting decoder");
 
-  if (self->started)
-    gst_omx_video_dec_drain (self);
+  gst_omx_video_dec_drain (self);
 
   /* FIXME: Workaround for 
    * https://bugzilla.gnome.org/show_bug.cgi?id=654529
@@ -1214,6 +1219,7 @@ gst_omx_video_dec_reset (GstBaseVideoDecoder * decoder)
   }
 
   /* Start the srcpad loop again */
+  self->eos = FALSE;
   self->downstream_flow_ret = GST_FLOW_OK;
   gst_pad_start_task (GST_BASE_VIDEO_CODEC_SRC_PAD (self),
       (GstTaskFunction) gst_omx_video_dec_loop, decoder);
@@ -1246,6 +1252,11 @@ gst_omx_video_dec_handle_frame (GstBaseVideoDecoder * decoder,
 
   GST_DEBUG_OBJECT (self, "Handling frame");
 
+  if (self->eos) {
+    GST_WARNING_OBJECT (self, "Got frame after EOS");
+    return GST_FLOW_UNEXPECTED;
+  }
+
   timestamp = frame->presentation_timestamp;
   duration = frame->presentation_duration;
 
@@ -1430,6 +1441,11 @@ gst_omx_video_dec_finish (GstBaseVideoDecoder * decoder)
 
   GST_DEBUG_OBJECT (self, "Sending EOS to the component");
 
+  /* Don't send EOS buffer twice, this doesn't work */
+  if (self->eos)
+    return GST_BASE_VIDEO_DECODER_FLOW_DROPPED;
+  self->eos = TRUE;
+
   /* Send an EOS buffer to the component and let the base
    * class drop the EOS event. We will send it later when
    * the EOS buffer arrives on the output port. */
@@ -1450,7 +1466,13 @@ gst_omx_video_dec_drain (GstOMXVideoDec * self)
 
   GST_DEBUG_OBJECT (self, "Draining component");
 
-  g_assert (self->started);
+  if (!self->started)
+    return GST_FLOW_OK;
+  self->started = FALSE;
+
+  /* Don't send EOS buffer twice, this doesn't work */
+  if (self->eos)
+    return GST_FLOW_OK;
 
   /* Send an EOS buffer to the component and let the base
    * class drop the EOS event. We will send it later when
@@ -1470,5 +1492,7 @@ gst_omx_video_dec_drain (GstOMXVideoDec * self)
   g_mutex_unlock (self->drain_lock);
   GST_BASE_VIDEO_CODEC_STREAM_LOCK (self);
 
+  self->started = FALSE;
+
   return GST_FLOW_OK;
 }
diff --git a/omx/gstomxvideodec.h b/omx/gstomxvideodec.h
index 5d3fd32c15..afbd4db2cf 100644
--- a/omx/gstomxvideodec.h
+++ b/omx/gstomxvideodec.h
@@ -65,6 +65,9 @@ struct _GstOMXVideoDec
   /* TRUE if EOS buffers shouldn't be forwarded */
   gboolean draining;
 
+  /* TRUE if upstream is EOS */
+  gboolean eos;
+
   GstFlowReturn downstream_flow_ret;
 };