diff --git a/gst/encoding/gstencodebasebin.c b/gst/encoding/gstencodebasebin.c
index e5a69c8f2d..0054433449 100644
--- a/gst/encoding/gstencodebasebin.c
+++ b/gst/encoding/gstencodebasebin.c
@@ -1196,16 +1196,19 @@ _capsfilter_force_format (GstPad * pad,
     if (sgroup->smart_capsfilter) {
       GstStructure *structure = gst_caps_get_structure (caps, 0);
 
-      /* Pick a stream format that allows for in-band SPS updates, and remove
-       * restrictions on fields that can be updated by codec_data or in-band SPS
+      /* Pick a stream format that allows for in-band SPS updates if none
+       * specified by the user, and remove restrictions on fields that can be
+       * updated by codec_data or in-band SPS
        */
-      if (gst_structure_has_name (structure, "video/x-h264")) {
+      if (gst_structure_has_name (structure, "video/x-h264") &&
+          !gst_structure_has_field (structure, "stream_format")) {
         gst_structure_set (structure, "stream-format",
             G_TYPE_STRING, "avc3", NULL);
 
         gst_structure_remove_fields (structure, "codec_data", "profile",
             "level", NULL);
-      } else if (gst_structure_has_name (structure, "video/x-h265")) {
+      } else if (gst_structure_has_name (structure, "video/x-h265") &&
+          !gst_structure_has_field (structure, "stream_format")) {
         gst_structure_set (structure, "stream-format",
             G_TYPE_STRING, "hev1", NULL);
 
diff --git a/gst/encoding/gstsmartencoder.c b/gst/encoding/gstsmartencoder.c
index b647b8cca4..18245973f1 100644
--- a/gst/encoding/gstsmartencoder.c
+++ b/gst/encoding/gstsmartencoder.c
@@ -681,13 +681,14 @@ smart_encoder_sink_query (GstPad * pad, GstObject * ghostpad, GstQuery * query)
 static gboolean
 gst_smart_encoder_add_parser (GstSmartEncoder * self, GstCaps * format)
 {
+  const gchar *stream_format;
   GstPad *chainpad, *internal_chainpad, *sinkpad;
+  GstStructure *structure = gst_caps_get_structure (format, 0);
   GstElement *capsfilter = gst_element_factory_make ("capsfilter", NULL);
 
   gst_bin_add (GST_BIN (self), capsfilter);
   g_object_set (capsfilter, "caps", format, NULL);
-  if (gst_structure_has_name (gst_caps_get_structure (format, 0),
-          "video/x-h264")) {
+  if (gst_structure_has_name (structure, "video/x-h264")) {
     GstElement *parser = gst_element_factory_make ("h264parse", NULL);
     if (!parser) {
       GST_ERROR_OBJECT (self, "`h264parse` is missing, can't encode smartly");
@@ -695,7 +696,9 @@ gst_smart_encoder_add_parser (GstSmartEncoder * self, GstCaps * format)
       goto failed;
     }
 
-    g_object_set (parser, "config-interval", -1, NULL);
+    stream_format = gst_structure_get_string (structure, "stream-format");
+    if (g_strcmp0 (stream_format, "avc1"))
+      g_object_set (parser, "config-interval", -1, NULL);
 
     if (!gst_bin_add (GST_BIN (self), parser)) {
       GST_ERROR_OBJECT (self, "Could not add parser.");
@@ -719,7 +722,9 @@ gst_smart_encoder_add_parser (GstSmartEncoder * self, GstCaps * format)
       goto failed;
     }
 
-    g_object_set (parser, "config-interval", -1, NULL);
+    stream_format = gst_structure_get_string (structure, "stream-format");
+    if (g_strcmp0 (stream_format, "hvc1"))
+      g_object_set (parser, "config-interval", -1, NULL);
 
     if (!gst_bin_add (GST_BIN (self), parser)) {
       GST_ERROR_OBJECT (self, "Could not add parser.");