diff --git a/gst/udp/gstudpsrc.c b/gst/udp/gstudpsrc.c
index e68401203e..bd7082d470 100644
--- a/gst/udp/gstudpsrc.c
+++ b/gst/udp/gstudpsrc.c
@@ -382,13 +382,20 @@ retry:
     goto no_select;
 
   do {
+    gint64 timeout;
+
     try_again = FALSE;
 
-    GST_LOG_OBJECT (udpsrc, "doing select, timeout %" G_GUINT64_FORMAT,
-        udpsrc->timeout);
+    if (udpsrc->timeout)
+      timeout = udpsrc->timeout / 1000;
+    else
+      timeout = -1;
 
-    if (!g_socket_condition_wait (udpsrc->used_socket, G_IO_IN | G_IO_PRI,
-            udpsrc->cancellable, &err)) {
+    GST_LOG_OBJECT (udpsrc, "doing select, timeout %" G_GUINT64_FORMAT,
+        timeout);
+
+    if (!g_socket_condition_timed_wait (udpsrc->used_socket, G_IO_IN | G_IO_PRI,
+            timeout, udpsrc->cancellable, &err)) {
       if (g_error_matches (err, G_IO_ERROR, G_IO_ERROR_BUSY)
           || g_error_matches (err, G_IO_ERROR, G_IO_ERROR_CANCELLED)) {
         goto stopped;
@@ -808,9 +815,6 @@ gst_udpsrc_start (GstBaseSrc * bsrc)
       goto getsockname_error;
   }
 
-  if (src->timeout)
-    g_socket_set_timeout (src->used_socket, src->timeout / GST_SECOND);
-
 #ifdef SO_RCVBUF
   {
     gint rcvsize, ret;