diff --git a/gst/tcp/gstmultihandlesink.c b/gst/tcp/gstmultihandlesink.c
index 6ddbc981bc..651d0ac080 100644
--- a/gst/tcp/gstmultihandlesink.c
+++ b/gst/tcp/gstmultihandlesink.c
@@ -114,6 +114,8 @@
 #include <netinet/in.h>
 #endif
 
+#include <string.h>
+
 #define NOT_IMPLEMENTED 0
 
 static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE ("sink",
@@ -1140,13 +1142,9 @@ gst_multi_handle_sink_client_queue_buffer (GstMultiHandleSink * mhsink,
 static gboolean
 is_sync_frame (GstMultiHandleSink * sink, GstBuffer * buffer)
 {
-  if (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT)) {
+  if (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT))
     return FALSE;
-  } else if (!GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_HEADER)) {
-    return TRUE;
-  }
-
-  return FALSE;
+  return TRUE;
 }
 
 /* find the keyframe in the list of buffers starting the
@@ -1844,10 +1842,64 @@ restart:
   }
 }
 
+static gboolean
+buffer_is_in_caps (GstMultiHandleSink * sink, GstBuffer * buf)
+{
+  GstCaps *caps;
+  GstStructure *s;
+  const GValue *v;
+
+  caps = gst_pad_get_current_caps (GST_BASE_SINK_PAD (sink));
+  if (!caps)
+    return FALSE;
+  s = gst_caps_get_structure (caps, 0);
+  if (!gst_structure_has_field (s, "streamheader")) {
+    gst_caps_unref (caps);
+    return FALSE;
+  }
+
+  v = gst_structure_get_value (s, "streamheader");
+  if (GST_VALUE_HOLDS_ARRAY (v)) {
+    guint n = gst_value_array_get_size (v);
+    guint i;
+    GstMapInfo map;
+
+    gst_buffer_map (buf, &map, GST_MAP_READ);
+
+    for (i = 0; i < n; i++) {
+      const GValue *v2 = gst_value_array_get_value (v, i);
+      GstBuffer *buf2;
+      GstMapInfo map2;
+
+      if (!GST_VALUE_HOLDS_BUFFER (v2))
+        continue;
+
+      buf2 = gst_value_get_buffer (v2);
+      if (buf == buf2) {
+        gst_caps_unref (caps);
+        return TRUE;
+      }
+      gst_buffer_map (buf2, &map2, GST_MAP_READ);
+      if (map.size == map2.size && memcmp (map.data, map2.data, map.size) == 0) {
+        gst_buffer_unmap (buf2, &map2);
+        gst_buffer_unmap (buf, &map);
+        gst_caps_unref (caps);
+        return TRUE;
+      }
+      gst_buffer_unmap (buf2, &map2);
+    }
+    gst_buffer_unmap (buf, &map);
+  }
+
+  gst_caps_unref (caps);
+
+  return FALSE;
+}
+
 static GstFlowReturn
 gst_multi_handle_sink_render (GstBaseSink * bsink, GstBuffer * buf)
 {
-  gboolean in_caps;
+  gboolean is_header, in_caps;
 #if 0
   GstCaps *bufcaps, *padcaps;
 #endif
@@ -1868,8 +1920,9 @@ gst_multi_handle_sink_render (GstBaseSink * bsink, GstBuffer * buf)
     goto no_caps;
 #endif
 
-  /* get IN_CAPS first, code below might mess with the flags */
-  in_caps = GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_HEADER);
+  /* get HEADER first, code below might mess with the flags */
+  is_header = GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_HEADER);
+  in_caps = is_header && buffer_is_in_caps (sink, buf);
 
 #if 0
   /* stamp the buffer with previous caps if no caps set */
@@ -1909,7 +1962,7 @@ gst_multi_handle_sink_render (GstBaseSink * bsink, GstBuffer * buf)
   /* if we get IN_CAPS buffers, but the previous buffer was not IN_CAPS,
    * it means we're getting new streamheader buffers, and we should clear
    * the old ones */
-  if (in_caps && !sink->previous_buffer_in_caps) {
+  if (is_header && !sink->previous_buffer_is_header) {
     GST_DEBUG_OBJECT (sink,
         "receiving new HEADER buffers, clearing old streamheader");
     g_slist_foreach (sink->streamheader, (GFunc) gst_mini_object_unref, NULL);
@@ -1918,7 +1971,7 @@ gst_multi_handle_sink_render (GstBaseSink * bsink, GstBuffer * buf)
   }
 
   /* save the current in_caps */
-  sink->previous_buffer_in_caps = in_caps;
+  sink->previous_buffer_is_header = is_header;
 
   /* if the incoming buffer is marked as IN CAPS, then we assume for now
    * it's a streamheader that needs to be sent to each new client, so we
diff --git a/gst/tcp/gstmultihandlesink.h b/gst/tcp/gstmultihandlesink.h
index bd125eb2c9..e64d44a4c2 100644
--- a/gst/tcp/gstmultihandlesink.h
+++ b/gst/tcp/gstmultihandlesink.h
@@ -203,7 +203,7 @@ struct _GstMultiHandleSink {
   GCancellable *cancellable;
 
   GSList *streamheader; /* GSList of GstBuffers to use as streamheader */
-  gboolean previous_buffer_in_caps;
+  gboolean previous_buffer_is_header;
 
   gint qos_dscp;