diff --git a/subprojects/gst-libav/ext/libav/gstavauddec.c b/subprojects/gst-libav/ext/libav/gstavauddec.c
index a24f7d5bae..597cb547bc 100644
--- a/subprojects/gst-libav/ext/libav/gstavauddec.c
+++ b/subprojects/gst-libav/ext/libav/gstavauddec.c
@@ -773,8 +773,10 @@ gst_ffmpegauddec_handle_frame (GstAudioDecoder * decoder, GstBuffer * inbuf)
   }
 
   if (avcodec_send_packet (ffmpegdec->context, &packet) < 0) {
+    av_packet_free_side_data (&packet);
     goto send_packet_failed;
   }
+  av_packet_free_side_data (&packet);
 
   do {
     /* decode a frame of audio now */
diff --git a/subprojects/gst-libav/ext/libav/gstavviddec.c b/subprojects/gst-libav/ext/libav/gstavviddec.c
index 6d7c4cd0de..336da5bd3a 100644
--- a/subprojects/gst-libav/ext/libav/gstavviddec.c
+++ b/subprojects/gst-libav/ext/libav/gstavviddec.c
@@ -2119,6 +2119,9 @@ gst_ffmpegviddec_handle_frame (GstVideoDecoder * decoder,
   /* now decode the frame */
   gst_avpacket_init (&packet, data, size);
 
+  if (!packet.size)
+    goto done;
+
   if (ffmpegdec->palette) {
     guint8 *pal;
 
@@ -2128,9 +2131,6 @@ gst_ffmpegviddec_handle_frame (GstVideoDecoder * decoder,
     GST_DEBUG_OBJECT (ffmpegdec, "copy pal %p %p", &packet, pal);
   }
 
-  if (!packet.size)
-    goto done;
-
   /* save reference to the timing info */
   ffmpegdec->context->reordered_opaque = (gint64) frame->system_frame_number;
   ffmpegdec->picture->reordered_opaque = (gint64) frame->system_frame_number;
@@ -2146,8 +2146,10 @@ gst_ffmpegviddec_handle_frame (GstVideoDecoder * decoder,
   GST_VIDEO_DECODER_STREAM_UNLOCK (ffmpegdec);
   if (avcodec_send_packet (ffmpegdec->context, &packet) < 0) {
     GST_VIDEO_DECODER_STREAM_LOCK (ffmpegdec);
+    av_packet_free_side_data (&packet);
     goto send_packet_failed;
   }
+  av_packet_free_side_data (&packet);
   GST_VIDEO_DECODER_STREAM_LOCK (ffmpegdec);
 
   do {