diff --git a/gst/udp/gstudpsrc.c b/gst/udp/gstudpsrc.c
index 36ff1d6dd6..f91fc716a0 100644
--- a/gst/udp/gstudpsrc.c
+++ b/gst/udp/gstudpsrc.c
@@ -864,6 +864,10 @@ gst_udpsrc_create (GstPushSrc * psrc, GstBuffer ** buf)
   p_saddr = (udpsrc->retrieve_sender_address) ? &saddr : NULL;
 
 retry:
+  if (saddr != NULL) {
+    g_object_unref (saddr);
+    saddr = NULL;
+  }
 
   do {
     gint64 timeout;
@@ -897,11 +901,6 @@ retry:
     }
   } while (G_UNLIKELY (try_again));
 
-  if (saddr != NULL) {
-    g_object_unref (saddr);
-    saddr = NULL;
-  }
-
   res =
       g_socket_receive_message (udpsrc->used_socket, p_saddr, udpsrc->vec, 2,
       p_msgs, &n_msgs, &flags, udpsrc->cancellable, &err);
@@ -973,12 +972,6 @@ retry:
     if (skip_packet) {
       GST_DEBUG_OBJECT (udpsrc,
           "Dropping packet for a different multicast address");
-
-      if (saddr != NULL) {
-        g_object_unref (saddr);
-        saddr = NULL;
-      }
-
       goto retry;
     }
   }