diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph264pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph264pay.c
index 3d51076bc8..6c12b6c579 100644
--- a/subprojects/gst-plugins-good/gst/rtp/gstrtph264pay.c
+++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph264pay.c
@@ -1404,58 +1404,18 @@ gst_rtp_h264_pay_handle_buffer (GstRTPBasePayload * basepayload,
 
   avc = rtph264pay->stream_format == GST_H264_STREAM_FORMAT_AVC;
 
-  if (avc) {
-    /* In AVC mode, there is no adapter, so nothing to drain */
-    if (draining)
-      return GST_FLOW_OK;
-  } else {
-    if (buffer) {
-      if (!GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT)) {
-        if (gst_adapter_available (rtph264pay->adapter) == 0)
-          rtph264pay->delta_unit = FALSE;
-        else
-          /* This buffer contains a key frame but the adapter isn't empty. So
-           * we'll purge it first by sending a first packet and then the second
-           * one won't have the DELTA_UNIT flag. */
-          delayed_not_delta_unit = TRUE;
-      }
-
-      if (GST_BUFFER_IS_DISCONT (buffer)) {
-        if (gst_adapter_available (rtph264pay->adapter) == 0)
-          rtph264pay->discont = TRUE;
-        else
-          /* This buffer has the DISCONT flag but the adapter isn't empty. So
-           * we'll purge it first by sending a first packet and then the second
-           * one will have the DISCONT flag set. */
-          delayed_discont = TRUE;
-      }
-
-      marker = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_MARKER);
-      gst_adapter_push (rtph264pay->adapter, buffer);
-      buffer = NULL;
-    }
-
-    /* We want to use the first TS used to construct the following NAL */
-    dts = gst_adapter_prev_dts (rtph264pay->adapter, NULL);
-    pts = gst_adapter_prev_pts (rtph264pay->adapter, NULL);
-
-    size = gst_adapter_available (rtph264pay->adapter);
-    /* Nothing to do here if the adapter is empty, e.g. on EOS */
-    if (size == 0)
-      return GST_FLOW_OK;
-    data = gst_adapter_map (rtph264pay->adapter, size);
-    GST_DEBUG_OBJECT (basepayload, "got %" G_GSIZE_FORMAT " bytes", size);
-  }
-
   ret = GST_FLOW_OK;
 
-  /* now loop over all NAL units and put them in a packet */
   if (avc) {
     GstBufferMemoryMap memory;
     gsize remaining_buffer_size;
     guint nal_length_size;
     gsize offset = 0;
 
+    /* In AVC mode, there is no adapter, so nothing to drain */
+    if (draining)
+      return GST_FLOW_OK;
+
     gst_buffer_memory_map (buffer, &memory);
     remaining_buffer_size = gst_buffer_get_size (buffer);
 
@@ -1534,6 +1494,43 @@ gst_rtp_h264_pay_handle_buffer (GstRTPBasePayload * basepayload,
     guint next;
     gboolean update = FALSE;
 
+    if (buffer) {
+      if (!GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT)) {
+        if (gst_adapter_available (rtph264pay->adapter) == 0)
+          rtph264pay->delta_unit = FALSE;
+        else
+          /* This buffer contains a key frame but the adapter isn't empty. So
+           * we'll purge it first by sending a first packet and then the second
+           * one won't have the DELTA_UNIT flag. */
+          delayed_not_delta_unit = TRUE;
+      }
+
+      if (GST_BUFFER_IS_DISCONT (buffer)) {
+        if (gst_adapter_available (rtph264pay->adapter) == 0)
+          rtph264pay->discont = TRUE;
+        else
+          /* This buffer has the DISCONT flag but the adapter isn't empty. So
+           * we'll purge it first by sending a first packet and then the second
+           * one will have the DISCONT flag set. */
+          delayed_discont = TRUE;
+      }
+
+      marker = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_MARKER);
+      gst_adapter_push (rtph264pay->adapter, buffer);
+      buffer = NULL;
+    }
+
+    /* We want to use the first TS used to construct the following NAL */
+    dts = gst_adapter_prev_dts (rtph264pay->adapter, NULL);
+    pts = gst_adapter_prev_pts (rtph264pay->adapter, NULL);
+
+    size = gst_adapter_available (rtph264pay->adapter);
+    /* Nothing to do here if the adapter is empty, e.g. on EOS */
+    if (size == 0)
+      return GST_FLOW_OK;
+    data = gst_adapter_map (rtph264pay->adapter, size);
+    GST_DEBUG_OBJECT (basepayload, "got %" G_GSIZE_FORMAT " bytes", size);
+
     /* get offset of first start code */
     next = next_start_code (data, size);
 
diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.c
index 5a08f8bbe2..eae0ad443b 100644
--- a/subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.c
+++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.c
@@ -1488,40 +1488,8 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload,
   hevc = (rtph265pay->stream_format == GST_H265_STREAM_FORMAT_HEV1)
       || (rtph265pay->stream_format == GST_H265_STREAM_FORMAT_HVC1);
 
-  if (hevc) {
-    /* In hevc mode, there is no adapter, so nothing to drain */
-    if (draining)
-      return GST_FLOW_OK;
-  } else {
-    if (buffer) {
-      if (!GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT)) {
-        if (gst_adapter_available (rtph265pay->adapter) == 0)
-          rtph265pay->delta_unit = FALSE;
-        else
-          delayed_not_delta_unit = TRUE;
-      }
-
-      discont = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DISCONT);
-      marker = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_MARKER);
-      gst_adapter_push (rtph265pay->adapter, buffer);
-      buffer = NULL;
-    }
-
-    /* We want to use the first TS used to construct the following NAL */
-    dts = gst_adapter_prev_dts (rtph265pay->adapter, NULL);
-    pts = gst_adapter_prev_pts (rtph265pay->adapter, NULL);
-
-    size = gst_adapter_available (rtph265pay->adapter);
-    /* Nothing to do here if the adapter is empty, e.g. on EOS */
-    if (size == 0)
-      return GST_FLOW_OK;
-    data = gst_adapter_map (rtph265pay->adapter, size);
-    GST_DEBUG_OBJECT (basepayload, "got %" G_GSIZE_FORMAT " bytes", size);
-  }
-
   ret = GST_FLOW_OK;
 
-  /* now loop over all NAL units and put them in a packet */
   if (hevc) {
     GstBufferMemoryMap memory;
     gsize remaining_buffer_size;
@@ -1529,6 +1497,10 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload,
     gsize offset = 0;
     GPtrArray *paybufs;
 
+    /* In hevc mode, there is no adapter, so nothing to drain */
+    if (draining)
+      return GST_FLOW_OK;
+
     paybufs = g_ptr_array_new ();
     nal_length_size = rtph265pay->nal_length_size;
 
@@ -1609,6 +1581,31 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload,
     gboolean update = FALSE;
     GPtrArray *paybufs;
 
+    if (buffer) {
+      if (!GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT)) {
+        if (gst_adapter_available (rtph265pay->adapter) == 0)
+          rtph265pay->delta_unit = FALSE;
+        else
+          delayed_not_delta_unit = TRUE;
+      }
+
+      discont = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DISCONT);
+      marker = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_MARKER);
+      gst_adapter_push (rtph265pay->adapter, buffer);
+      buffer = NULL;
+    }
+
+    /* We want to use the first TS used to construct the following NAL */
+    dts = gst_adapter_prev_dts (rtph265pay->adapter, NULL);
+    pts = gst_adapter_prev_pts (rtph265pay->adapter, NULL);
+
+    size = gst_adapter_available (rtph265pay->adapter);
+    /* Nothing to do here if the adapter is empty, e.g. on EOS */
+    if (size == 0)
+      return GST_FLOW_OK;
+    data = gst_adapter_map (rtph265pay->adapter, size);
+    GST_DEBUG_OBJECT (basepayload, "got %" G_GSIZE_FORMAT " bytes", size);
+
     /* get offset of first start code */
     next = next_start_code (data, size);