diff --git a/gst/isomp4/gstqtmux.c b/gst/isomp4/gstqtmux.c
index e61ec6ce04..80d6baa2b5 100644
--- a/gst/isomp4/gstqtmux.c
+++ b/gst/isomp4/gstqtmux.c
@@ -1691,10 +1691,29 @@ serialize_error:
   }
 }
 
+static gboolean
+gst_qt_mux_downstream_is_seekable (GstQTMux * qtmux)
+{
+  gboolean seekable = FALSE;
+  GstQuery *query = gst_query_new_seeking (GST_FORMAT_BYTES);
+
+  if (gst_pad_peer_query (qtmux->srcpad, query)) {
+    gst_query_parse_seeking (query, NULL, &seekable, NULL, NULL);
+    GST_INFO_OBJECT (qtmux, "downstream is %sseekable", seekable ? "" : "not ");
+  } else {
+    /* have to assume seeking is not supported if query not handled downstream */
+    GST_WARNING_OBJECT (qtmux, "downstream did not handle seeking query");
+    seekable = FALSE;
+  }
+  gst_query_unref (query);
+
+  return seekable;
+}
+
+
 static GstFlowReturn
 gst_qt_mux_start_file (GstQTMux * qtmux)
 {
-  GstQTMuxClass *qtmux_klass = (GstQTMuxClass *) (G_OBJECT_GET_CLASS (qtmux));
   GstFlowReturn ret = GST_FLOW_OK;
   GstCaps *caps;
   GstSegment segment;
@@ -1713,30 +1732,15 @@ gst_qt_mux_start_file (GstQTMux * qtmux)
   gst_pad_set_caps (qtmux->srcpad, caps);
   gst_caps_unref (caps);
 
-  /* if not streaming, check if downstream is seekable */
+  /* if not streaming or doing fast-start, check if downstream is seekable */
   if (!qtmux->streamable) {
-    gboolean seekable;
-    GstQuery *query;
-
-    query = gst_query_new_seeking (GST_FORMAT_BYTES);
-    if (gst_pad_peer_query (qtmux->srcpad, query)) {
-      gst_query_parse_seeking (query, NULL, &seekable, NULL, NULL);
-      GST_INFO_OBJECT (qtmux, "downstream is %sseekable",
-          seekable ? "" : "not ");
-    } else {
-      /* have to assume seeking is supported if query not handled downstream */
-      GST_WARNING_OBJECT (qtmux, "downstream did not handle seeking query");
-      seekable = FALSE;
-    }
-    gst_query_unref (query);
-    if (!seekable) {
-      if (qtmux_klass->format != GST_QT_MUX_FORMAT_ISML) {
+    if (!gst_qt_mux_downstream_is_seekable (qtmux)) {
+      if (qtmux->fragment_duration == 0) {
         if (!qtmux->fast_start) {
-          GST_ELEMENT_WARNING (qtmux, STREAM, FAILED,
-              ("Downstream is not seekable and headers can't be rewritten"),
+          GST_ELEMENT_ERROR (qtmux, STREAM, MUX,
+              ("Downstream is not seekable - will not be able to create a playable file"),
               (NULL));
-          /* FIXME: Is there something better we can do? */
-          qtmux->streamable = TRUE;
+          return GST_FLOW_ERROR;
         }
       } else {
         GST_WARNING_OBJECT (qtmux, "downstream is not seekable, but "
diff --git a/gst/isomp4/gstqtmux.h b/gst/isomp4/gstqtmux.h
index a9ea362a5a..19467dc884 100644
--- a/gst/isomp4/gstqtmux.h
+++ b/gst/isomp4/gstqtmux.h
@@ -196,6 +196,9 @@ struct _GstQTMux
   gchar *fast_start_file_path;
   gchar *moov_recov_file_path;
   guint32 fragment_duration;
+  /* Whether or not to work in 'streamable' mode and not
+   * seek to rewrite headers - only valid for fragmented
+   * mode. */
   gboolean streamable;
 
   /* for request pad naming */
diff --git a/tests/check/elements/qtmux.c b/tests/check/elements/qtmux.c
index 105b1e80db..199ff1cbae 100644
--- a/tests/check/elements/qtmux.c
+++ b/tests/check/elements/qtmux.c
@@ -153,8 +153,24 @@ teardown_src_pad (GstPad * srcpad)
   gst_object_unref (srcpad);
 }
 
+gboolean downstream_is_seekable;
+static gboolean
+qtmux_sinkpad_query (GstPad * pad, GstObject * parent, GstQuery * query)
+{
+  gboolean ret = FALSE;
+
+  if (GST_QUERY_TYPE (query) == GST_QUERY_SEEKING) {
+    gst_query_set_seeking (query, GST_FORMAT_BYTES, downstream_is_seekable, 0,
+        -1);
+    ret = TRUE;
+  }
+
+  return ret;
+}
+
 static GstElement *
-setup_qtmux (GstStaticPadTemplate * srctemplate, const gchar * sinkname)
+setup_qtmux (GstStaticPadTemplate * srctemplate, const gchar * sinkname,
+    gboolean seekable)
 {
   GstElement *qtmux;
 
@@ -162,6 +178,10 @@ setup_qtmux (GstStaticPadTemplate * srctemplate, const gchar * sinkname)
   qtmux = gst_check_setup_element ("qtmux");
   mysrcpad = setup_src_pad (qtmux, srctemplate, sinkname);
   mysinkpad = gst_check_setup_sink_pad (qtmux, &sinktemplate);
+
+  downstream_is_seekable = seekable;
+  gst_pad_set_query_function (mysinkpad, qtmux_sinkpad_query);
+
   gst_pad_set_active (mysrcpad, TRUE);
   gst_pad_set_active (mysinkpad, TRUE);
 
@@ -195,7 +215,7 @@ check_qtmux_pad (GstStaticPadTemplate * srctemplate, const gchar * sinkname,
   guint8 data2[4] = "moov";
   GstSegment segment;
 
-  qtmux = setup_qtmux (srctemplate, sinkname);
+  qtmux = setup_qtmux (srctemplate, sinkname, TRUE);
   g_object_set (qtmux, "dts-method", dts_method, NULL);
   fail_unless (gst_element_set_state (qtmux,
           GST_STATE_PLAYING) == GST_STATE_CHANGE_SUCCESS,
@@ -285,7 +305,7 @@ check_qtmux_pad_fragmented (GstStaticPadTemplate * srctemplate,
   guint8 data4[4] = "mfra";
   GstSegment segment;
 
-  qtmux = setup_qtmux (srctemplate, sinkname);
+  qtmux = setup_qtmux (srctemplate, sinkname, !streamable);
   g_object_set (qtmux, "dts-method", dts_method, NULL);
   g_object_set (qtmux, "fragment-duration", 2000, NULL);
   g_object_set (qtmux, "streamable", streamable, NULL);
@@ -515,7 +535,7 @@ GST_END_TEST;
 
 GST_START_TEST (test_reuse)
 {
-  GstElement *qtmux = setup_qtmux (&srcvideotemplate, "video_%u");
+  GstElement *qtmux = setup_qtmux (&srcvideotemplate, "video_%u", TRUE);
   GstBuffer *inbuffer;
   GstCaps *caps;
   GstSegment segment;