diff --git a/ext/srtp/gstsrtpdec.c b/ext/srtp/gstsrtpdec.c
index 2e4fbf25da..04d1883f5c 100644
--- a/ext/srtp/gstsrtpdec.c
+++ b/ext/srtp/gstsrtpdec.c
@@ -396,12 +396,6 @@ get_stream_from_caps (GstSrtpDec * filter, GstCaps * caps, guint32 ssrc)
   if (!rtp_cipher || !rtp_auth || !rtcp_cipher || !rtcp_auth)
     goto error;
 
-  if (!gst_structure_get (s, "srtp-key", GST_TYPE_BUFFER, &buf, NULL) || !buf) {
-    goto error;
-  } else {
-    GST_DEBUG ("Got key [%p]", buf);
-    stream->key = buf;
-  }
 
   stream->rtp_cipher = enum_value_from_nick (GST_TYPE_SRTP_CIPHER_TYPE,
       rtp_cipher);
@@ -424,6 +418,16 @@ get_stream_from_caps (GstSrtpDec * filter, GstCaps * caps, guint32 ssrc)
     goto error;
   }
 
+  if (gst_structure_get (s, "srtp-key", GST_TYPE_BUFFER, &buf, NULL) || !buf) {
+    GST_DEBUG ("Got key [%p]", buf);
+    stream->key = buf;
+  } else if (stream->rtp_cipher != GST_SRTP_CIPHER_NULL ||
+      stream->rtcp_cipher != GST_SRTP_CIPHER_NULL ||
+      stream->rtp_auth != GST_SRTP_AUTH_NULL ||
+      stream->rtcp_auth != GST_SRTP_AUTH_NULL) {
+    goto error;
+  }
+
   return stream;
 
 error:
@@ -455,6 +459,7 @@ init_session_stream (GstSrtpDec * filter, guint32 ssrc,
   err_status_t ret;
   srtp_policy_t policy;
   GstMapInfo map;
+  guchar tmp[1];
 
   memset (&policy, 0, sizeof (srtp_policy_t));
 
@@ -468,11 +473,15 @@ init_session_stream (GstSrtpDec * filter, guint32 ssrc,
   set_crypto_policy_cipher_auth (stream->rtcp_cipher, stream->rtcp_auth,
       &policy.rtcp);
 
-  gst_buffer_map (stream->key, &map, GST_MAP_READ);
+  if (stream->key) {
+    gst_buffer_map (stream->key, &map, GST_MAP_READ);
+    policy.key = (guchar *) map.data;
+  } else {
+    policy.key = tmp;
+  }
 
   policy.ssrc.value = ssrc;
   policy.ssrc.type = ssrc_specific;
-  policy.key = (guchar *) map.data;
   policy.next = NULL;
 
   /* If it is the first stream, create the session
@@ -483,7 +492,8 @@ init_session_stream (GstSrtpDec * filter, guint32 ssrc,
   else
     ret = srtp_add_stream (filter->session, &policy);
 
-  gst_buffer_unmap (stream->key, &map);
+  if (stream->key)
+    gst_buffer_unmap (stream->key, &map);
 
   if (ret == err_status_ok) {
     filter->first_session = FALSE;
@@ -547,7 +557,8 @@ update_session_stream_from_caps (GstSrtpDec * filter, guint32 ssrc,
     err = init_session_stream (filter, ssrc, stream);
 
     if (err != err_status_ok) {
-      gst_buffer_unref (stream->key);
+      if (stream->key)
+        gst_buffer_unref (stream->key);
       g_slice_free (GstSrtpDecSsrcStream, stream);
       stream = NULL;
     }
@@ -559,7 +570,8 @@ update_session_stream_from_caps (GstSrtpDec * filter, guint32 ssrc,
 static void
 clear_stream (GstSrtpDecSsrcStream * stream)
 {
-  gst_buffer_unref (stream->key);
+  if (stream->key)
+    gst_buffer_unref (stream->key);
   g_slice_free (GstSrtpDecSsrcStream, stream);
 }
 
@@ -628,7 +640,6 @@ gst_srtp_dec_sink_setcaps (GstPad * pad, GstObject * parent,
   ps = gst_caps_get_structure (caps, 0);
 
   if (gst_structure_has_field_typed (ps, "ssrc", G_TYPE_UINT) &&
-      gst_structure_has_field_typed (ps, "srtp-key", GST_TYPE_BUFFER) &&
       gst_structure_has_field_typed (ps, "srtp-cipher", G_TYPE_STRING) &&
       gst_structure_has_field_typed (ps, "srtp-auth", G_TYPE_STRING) &&
       gst_structure_has_field_typed (ps, "srtcp-cipher", G_TYPE_STRING) &&
diff --git a/ext/srtp/gstsrtpenc.c b/ext/srtp/gstsrtpenc.c
index d934c228bf..8ce2a6bc41 100644
--- a/ext/srtp/gstsrtpenc.c
+++ b/ext/srtp/gstsrtpenc.c
@@ -118,6 +118,10 @@ GST_DEBUG_CATEGORY_STATIC (gst_srtp_enc_debug);
 #define DEFAULT_RTCP_AUTH       DEFAULT_RTP_AUTH
 #define DEFAULT_RANDOM_KEY      FALSE
 
+#define HAS_CRYPTO(filter) (filter->rtp_cipher != GST_SRTP_CIPHER_NULL || \
+      filter->rtcp_cipher != GST_SRTP_CIPHER_NULL ||                      \
+      filter->rtp_auth != GST_SRTP_AUTH_NULL ||                           \
+      filter->rtcp_auth != GST_SRTP_AUTH_NULL)
 
 /* Filter signals and args */
 enum
@@ -320,6 +324,7 @@ check_new_stream_locked (GstSrtpEnc * filter, guint32 ssrc)
   err_status_t ret;
   srtp_policy_t policy;
   GstMapInfo map;
+  guchar tmp[1];
 
   memset (&policy, 0, sizeof (srtp_policy_t));
 
@@ -327,11 +332,25 @@ check_new_stream_locked (GstSrtpEnc * filter, guint32 ssrc)
   if (g_hash_table_lookup (filter->ssrcs_set, GUINT_TO_POINTER (ssrc)))
     return TRUE;
 
-  if (gst_buffer_get_size (filter->key) != SRTP_MASTER_KEY_LEN) {
-    GST_ELEMENT_ERROR (filter, LIBRARY, SETTINGS, ("Master key size is wrong"),
-        ("Expected master key of %d bytes, but received %" G_GSIZE_FORMAT
-            " bytes", SRTP_MASTER_KEY_LEN, gst_buffer_get_size (filter->key)));
-    return FALSE;
+  GST_OBJECT_LOCK (filter);
+
+  if (HAS_CRYPTO (filter)) {
+    if (filter->key == NULL) {
+      GST_OBJECT_UNLOCK (filter);
+      GST_ELEMENT_ERROR (filter, LIBRARY, SETTINGS,
+          ("Cipher is not NULL, key must be set"),
+          ("Cipher is not NULL, key must be set"));
+      return FALSE;
+    }
+    if (gst_buffer_get_size (filter->key) != SRTP_MASTER_KEY_LEN) {
+      GST_OBJECT_UNLOCK (filter);
+      GST_ELEMENT_ERROR (filter, LIBRARY, SETTINGS,
+          ("Master key size is wrong"),
+          ("Expected master key of %d bytes, but received %" G_GSIZE_FORMAT
+              " bytes", SRTP_MASTER_KEY_LEN,
+              gst_buffer_get_size (filter->key)));
+      return FALSE;
+    }
   }
 
   GST_DEBUG_OBJECT (filter, "Setting RTP/RTCP policy to %d / %d",
@@ -341,11 +360,15 @@ check_new_stream_locked (GstSrtpEnc * filter, guint32 ssrc)
   set_crypto_policy_cipher_auth (filter->rtcp_cipher, filter->rtcp_auth,
       &policy.rtcp);
 
-  gst_buffer_map (filter->key, &map, GST_MAP_READ);
+  if (HAS_CRYPTO (filter)) {
+    gst_buffer_map (filter->key, &map, GST_MAP_READ);
+    policy.key = (guchar *) map.data;
+  } else {
+    policy.key = tmp;
+  }
 
   policy.ssrc.value = ssrc;
   policy.ssrc.type = ssrc_specific;
-  policy.key = (guchar *) map.data;
   policy.next = NULL;
 
   /* If it is the first stream, create the session
@@ -356,11 +379,14 @@ check_new_stream_locked (GstSrtpEnc * filter, guint32 ssrc)
   else
     ret = srtp_add_stream (filter->session, &policy);
 
-  gst_buffer_unmap (filter->key, &map);
+  if (HAS_CRYPTO (filter))
+    gst_buffer_unmap (filter->key, &map);
 
   g_hash_table_insert (filter->ssrcs_set, GUINT_TO_POINTER (ssrc),
       GUINT_TO_POINTER (1));
 
+  GST_OBJECT_UNLOCK (filter);
+
   return ret == err_status_ok;
 }
 
@@ -688,8 +714,11 @@ gst_srtp_enc_sink_setcaps (GstPad * pad, GstSrtpEnc * filter,
 
   GST_OBJECT_LOCK (filter);
 
+  if (HAS_CRYPTO (filter))
+    gst_structure_set (ps, "srtp-key", GST_TYPE_BUFFER, filter->key, NULL);
+
   /* Add srtp-specific params to source caps */
-  gst_structure_set (ps, "srtp-key", GST_TYPE_BUFFER, filter->key,
+  gst_structure_set (ps,
       "srtp-cipher", G_TYPE_STRING,
       enum_nick_from_value (GST_TYPE_SRTP_CIPHER_TYPE, filter->rtp_cipher),
       "srtp-auth", G_TYPE_STRING,
@@ -1023,12 +1052,17 @@ gst_srtp_enc_change_state (GstElement * element, GstStateChange transition)
 
   switch (transition) {
     case GST_STATE_CHANGE_NULL_TO_READY:
-      if (!filter->key) {
-        if (filter->random_key) {
-          gst_srtp_enc_replace_random_key (filter);
-        } else {
-          GST_ERROR_OBJECT (element, "Need a key to get to READY");
-          return GST_STATE_CHANGE_FAILURE;
+      if (filter->rtp_cipher != GST_SRTP_CIPHER_NULL ||
+          filter->rtcp_cipher != GST_SRTP_CIPHER_NULL ||
+          filter->rtp_auth != GST_SRTP_AUTH_NULL ||
+          filter->rtcp_auth != GST_SRTP_AUTH_NULL) {
+        if (!filter->key) {
+          if (filter->random_key) {
+            gst_srtp_enc_replace_random_key (filter);
+          } else {
+            GST_ERROR_OBJECT (element, "Need a key to get to READY");
+            return GST_STATE_CHANGE_FAILURE;
+          }
         }
       }
       if ((filter->rtcp_cipher != NULL_CIPHER)