diff --git a/omx/gstomxvideoenc.c b/omx/gstomxvideoenc.c
index 138e510a72..9f4ffc83ae 100644
--- a/omx/gstomxvideoenc.c
+++ b/omx/gstomxvideoenc.c
@@ -1126,14 +1126,24 @@ gst_omx_video_enc_set_format (GstVideoEncoder * encoder,
 
     if (gst_omx_port_set_enabled (self->enc_in_port, FALSE) != OMX_ErrorNone)
       return FALSE;
+    if (gst_omx_port_set_enabled (self->enc_out_port, FALSE) != OMX_ErrorNone)
+      return FALSE;
     if (gst_omx_port_wait_buffers_released (self->enc_in_port,
             5 * GST_SECOND) != OMX_ErrorNone)
       return FALSE;
+    if (gst_omx_port_wait_buffers_released (self->enc_out_port,
+            1 * GST_SECOND) != OMX_ErrorNone)
+      return FALSE;
     if (gst_omx_port_deallocate_buffers (self->enc_in_port) != OMX_ErrorNone)
       return FALSE;
+    if (gst_omx_port_deallocate_buffers (self->enc_out_port) != OMX_ErrorNone)
+      return FALSE;
     if (gst_omx_port_wait_enabled (self->enc_in_port,
             1 * GST_SECOND) != OMX_ErrorNone)
       return FALSE;
+    if (gst_omx_port_wait_enabled (self->enc_out_port,
+            1 * GST_SECOND) != OMX_ErrorNone)
+      return FALSE;
 
     GST_DEBUG_OBJECT (self, "Encoder drained and disabled");
   }
@@ -1235,7 +1245,27 @@ gst_omx_video_enc_set_format (GstVideoEncoder * encoder,
       return FALSE;
     if (gst_omx_port_mark_reconfigured (self->enc_in_port) != OMX_ErrorNone)
       return FALSE;
+
+    /* If the output port is already configured with the new format we can
+     * allocate buffers here already */
+    if (self->enc_out_port->port_def.format.video.nFrameHeight == info->height
+        && self->enc_out_port->port_def.format.video.nFrameWidth ==
+        info->width) {
+      if (gst_omx_port_set_enabled (self->enc_out_port, TRUE) != OMX_ErrorNone)
+        return FALSE;
+      if (gst_omx_port_allocate_buffers (self->enc_out_port) != OMX_ErrorNone)
+        return FALSE;
+      if (gst_omx_port_wait_enabled (self->enc_out_port,
+              3 * GST_SECOND) != OMX_ErrorNone)
+        return FALSE;
+      if (gst_omx_port_populate (self->enc_out_port) != OMX_ErrorNone)
+        return FALSE;
+      if (gst_omx_port_mark_reconfigured (self->enc_out_port) != OMX_ErrorNone)
+        return FALSE;
+    }
   } else {
+    gboolean have_output_buffers = FALSE;
+
     if (gst_omx_component_set_state (self->enc, OMX_StateIdle) != OMX_ErrorNone)
       return FALSE;
 
@@ -1250,6 +1280,7 @@ gst_omx_video_enc_set_format (GstVideoEncoder * encoder,
         info->width) {
       if (gst_omx_port_allocate_buffers (self->enc_out_port) != OMX_ErrorNone)
         return FALSE;
+      have_output_buffers = TRUE;
     } else {
       /* And disable output port */
       if (gst_omx_port_set_enabled (self->enc_out_port, FALSE) != OMX_ErrorNone)
@@ -1268,8 +1299,12 @@ gst_omx_video_enc_set_format (GstVideoEncoder * encoder,
             OMX_StateExecuting) != OMX_ErrorNone)
       return FALSE;
 
-    if (gst_omx_port_populate (self->enc_out_port) != OMX_ErrorNone)
-      return FALSE;
+    if (have_output_buffers) {
+      if (gst_omx_port_populate (self->enc_out_port) != OMX_ErrorNone)
+        return FALSE;
+      if (gst_omx_port_mark_reconfigured (self->enc_out_port) != OMX_ErrorNone)
+        return FALSE;
+    }
 
     if (gst_omx_component_get_state (self->enc,
             GST_CLOCK_TIME_NONE) != OMX_StateExecuting)