diff --git a/gst/rtsp-server/rtsp-client.c b/gst/rtsp-server/rtsp-client.c
index 44a5c02641..6a5bc76654 100644
--- a/gst/rtsp-server/rtsp-client.c
+++ b/gst/rtsp-server/rtsp-client.c
@@ -337,6 +337,8 @@ find_media (GstRTSPClient * client, GstRTSPClientState * state)
       goto no_media;
 
     g_object_unref (factory);
+    factory = NULL;
+    state->factory = NULL;
 
     /* set ipv6 on the media before preparing */
     media->is_ipv6 = client->is_ipv6;
@@ -389,7 +391,6 @@ no_prepare:
   {
     send_generic_response (client, GST_RTSP_STS_SERVICE_UNAVAILABLE, state);
     g_object_unref (media);
-    g_object_unref (factory);
     return NULL;
   }
 }