From 3e7fb8339324b3601598ee07d65562420b3194f3 Mon Sep 17 00:00:00 2001 From: Sherrill Lin Date: Wed, 30 Jun 2021 16:01:10 -0400 Subject: [PATCH] webrtcstats: Improve selected candidate pair stats by adding ICE candidate info The implementation follows w3.org specs: * https://www.w3.org/TR/webrtc-stats/#icecandidate-dict* * https://www.w3.org/TR/webrtc-stats/#candidatepair-dict* Corresponding unit tests are also added. Rebased and updated from https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1462 Fixes #1207 Part-of: --- subprojects/gst-plugins-bad/ext/webrtc/fwd.h | 2 + .../gst-plugins-bad/ext/webrtc/gstwebrtcice.c | 158 +++++++++++++ .../gst-plugins-bad/ext/webrtc/gstwebrtcice.h | 21 ++ .../ext/webrtc/gstwebrtcstats.c | 214 ++++++++++++------ .../tests/check/elements/webrtcbin.c | 73 ++++++ 5 files changed, 400 insertions(+), 68 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/webrtc/fwd.h b/subprojects/gst-plugins-bad/ext/webrtc/fwd.h index aa26ec6dea..373d0321a2 100644 --- a/subprojects/gst-plugins-bad/ext/webrtc/fwd.h +++ b/subprojects/gst-plugins-bad/ext/webrtc/fwd.h @@ -57,6 +57,8 @@ typedef struct _TransportReceiveBinClass TransportReceiveBinClass; typedef struct _WebRTCTransceiver WebRTCTransceiver; typedef struct _WebRTCTransceiverClass WebRTCTransceiverClass; +typedef struct _GstWebRTCICECandidateStats GstWebRTCICECandidateStats; + G_END_DECLS #endif /* __WEBRTC_FWD_H__ */ diff --git a/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcice.c b/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcice.c index 207d29f2c2..8b1ce70b4c 100644 --- a/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcice.c +++ b/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcice.c @@ -848,6 +848,164 @@ gst_webrtc_ice_set_tos (GstWebRTCICE * ice, GstWebRTCICEStream * stream, nice_agent_set_stream_tos (ice->priv->nice_agent, item->nice_stream_id, tos); } +static const gchar * +_relay_type_to_string (GstUri * turn_server) +{ + const gchar *scheme; + const gchar *transport; + + if (!turn_server) + return "none"; + + scheme = gst_uri_get_scheme (turn_server); + transport = gst_uri_get_query_value (turn_server, "transport"); + + if (g_strcmp0 (scheme, "turns") == 0) { + return "tls"; + } else if (g_strcmp0 (scheme, "turn") == 0) { + if (!transport || g_strcmp0 (transport, "udp") == 0) + return "udp"; + if (!transport || g_strcmp0 (transport, "tcp") == 0) + return "tcp"; + } + + return "none"; +} + +static gchar * +_get_server_url (GstWebRTCICE * ice, NiceCandidate * cand) +{ + switch (cand->type) { + case NICE_CANDIDATE_TYPE_RELAYED: + return g_strdup (gst_uri_get_host (ice->turn_server)); + case NICE_CANDIDATE_TYPE_SERVER_REFLEXIVE: + return g_strdup (gst_uri_get_host (ice->stun_server)); + default: + return g_strdup (""); + } +} + +/* TODO: replace it with nice_candidate_type_to_string() + * when it's ready for use + * https://libnice.freedesktop.org/libnice/NiceCandidate.html#nice-candidate-type-to-string + */ +static const gchar * +_candidate_type_to_string (NiceCandidateType type) +{ + switch (type) { + case NICE_CANDIDATE_TYPE_HOST: + return "host"; + case NICE_CANDIDATE_TYPE_SERVER_REFLEXIVE: + return "srflx"; + case NICE_CANDIDATE_TYPE_PEER_REFLEXIVE: + return "prflx"; + case NICE_CANDIDATE_TYPE_RELAYED: + return "relay"; + default: + g_assert_not_reached (); + return NULL; + } +} + +static void +_populate_candidate_stats (GstWebRTCICE * ice, NiceCandidate * cand, + GstWebRTCICEStream * stream, GstWebRTCICECandidateStats * stats, + gboolean is_local) +{ + gchar ipaddr[INET6_ADDRSTRLEN]; + + g_assert (cand != NULL); + + nice_address_to_string (&cand->addr, ipaddr); + stats->port = nice_address_get_port (&cand->addr); + stats->ipaddr = g_strdup (ipaddr); + stats->stream_id = stream->stream_id; + stats->type = _candidate_type_to_string (cand->type); + stats->prio = cand->priority; + stats->proto = + cand->transport == NICE_CANDIDATE_TRANSPORT_UDP ? "udp" : "tcp"; + if (is_local) { + if (cand->type == NICE_CANDIDATE_TYPE_RELAYED) + stats->relay_proto = _relay_type_to_string (ice->turn_server); + stats->url = _get_server_url (ice, cand); + } +} + +static void +_populate_candidate_list_stats (GstWebRTCICE * ice, GSList * cands, + GstWebRTCICEStream * stream, GArray * result, gboolean is_local) +{ + GSList *item; + + for (item = cands; item != NULL; item = item->next) { + GstWebRTCICECandidateStats stats; + NiceCandidate *c = item->data; + _populate_candidate_stats (ice, c, stream, &stats, is_local); + g_array_append_val (result, stats); + } +} + +GArray * +gst_webrtc_ice_get_local_candidates (GstWebRTCICE * ice, + GstWebRTCICEStream * stream) +{ + GSList *cands = NULL; + + GArray *result = + g_array_new (FALSE, TRUE, sizeof (GstWebRTCICECandidateStats)); + + cands = nice_agent_get_local_candidates (ice->priv->nice_agent, + stream->stream_id, NICE_COMPONENT_TYPE_RTP); + + _populate_candidate_list_stats (ice, cands, stream, result, TRUE); + g_slist_free_full (cands, (GDestroyNotify) nice_candidate_free); + + return result; +} + +GArray * +gst_webrtc_ice_get_remote_candidates (GstWebRTCICE * ice, + GstWebRTCICEStream * stream) +{ + GSList *cands = NULL; + + GArray *result = + g_array_new (FALSE, TRUE, sizeof (GstWebRTCICECandidateStats)); + + cands = nice_agent_get_remote_candidates (ice->priv->nice_agent, + stream->stream_id, NICE_COMPONENT_TYPE_RTP); + + _populate_candidate_list_stats (ice, cands, stream, result, FALSE); + g_slist_free_full (cands, (GDestroyNotify) nice_candidate_free); + + return result; +} + +gboolean +gst_webrtc_ice_get_selected_pair (GstWebRTCICE * ice, + GstWebRTCICEStream * stream, GstWebRTCICECandidateStats ** local_stats, + GstWebRTCICECandidateStats ** remote_stats) +{ + NiceCandidate *local_cand = NULL; + NiceCandidate *remote_cand = NULL; + + if (stream) { + if (nice_agent_get_selected_pair (ice->priv->nice_agent, stream->stream_id, + NICE_COMPONENT_TYPE_RTP, &local_cand, &remote_cand)) { + *local_stats = g_new0 (GstWebRTCICECandidateStats, 1); + _populate_candidate_stats (ice, local_cand, stream, *local_stats, TRUE); + + *remote_stats = g_new0 (GstWebRTCICECandidateStats, 1); + _populate_candidate_stats (ice, remote_cand, stream, *remote_stats, + FALSE); + + return TRUE; + } + } + + return FALSE; +} + static void _clear_ice_stream (struct NiceStreamItem *item) { diff --git a/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcice.h b/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcice.h index 5e7543c80b..831ca1c9cb 100644 --- a/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcice.h +++ b/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcice.h @@ -56,6 +56,18 @@ struct _GstWebRTCICE guint max_rtp_port; }; +struct _GstWebRTCICECandidateStats +{ + gchar *ipaddr; + guint port; + guint stream_id; + const gchar *type; + const gchar *proto; + const gchar *relay_proto; + guint prio; + gchar *url; +}; + struct _GstWebRTCICEClass { GstObjectClass parent_class; @@ -107,6 +119,15 @@ void gst_webrtc_ice_set_on_ice_candidate (GstWebRTCIC void gst_webrtc_ice_set_tos (GstWebRTCICE * ice, GstWebRTCICEStream * stream, guint tos); + +GArray * gst_webrtc_ice_get_local_candidates (GstWebRTCICE * ice, + GstWebRTCICEStream * stream); +GArray * gst_webrtc_ice_get_remote_candidates (GstWebRTCICE * ice, + GstWebRTCICEStream * stream); +gboolean gst_webrtc_ice_get_selected_pair (GstWebRTCICE * ice, + GstWebRTCICEStream * stream, + GstWebRTCICECandidateStats ** local_stats, + GstWebRTCICECandidateStats ** remote_stats); G_END_DECLS #endif /* __GST_WEBRTC_ICE_H__ */ diff --git a/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcstats.c b/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcstats.c index 10074b9547..da3a4a6eda 100644 --- a/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcstats.c +++ b/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcstats.c @@ -26,6 +26,7 @@ #include "gstwebrtcstats.h" #include "gstwebrtcbin.h" +#include "icestream.h" #include "transportstream.h" #include "transportreceivebin.h" #include "utils.h" @@ -564,67 +565,138 @@ _get_stats_from_rtp_source_stats (GstWebRTCBin * webrtc, } } -/* https://www.w3.org/TR/webrtc-stats/#candidatepair-dict* */ +/* https://www.w3.org/TR/webrtc-stats/#icecandidate-dict* */ static gchar * -_get_stats_from_ice_transport (GstWebRTCBin * webrtc, - GstWebRTCICETransport * transport, const GstStructure * twcc_stats, - GstStructure * s) +_get_stats_from_ice_candidates (GstWebRTCBin * webrtc, + GstWebRTCICECandidateStats * can, const gchar * transport_id, + const gchar * candidate_tag, GstStructure * s) { GstStructure *stats; + GstWebRTCStatsType type; gchar *id; double ts; gst_structure_get_double (s, "timestamp", &ts); + id = g_strdup_printf ("ice-candidate-%s_%u_%s_%u", candidate_tag, + can->stream_id, can->ipaddr, can->port); + stats = gst_structure_new_empty (id); + + if (strcmp (candidate_tag, "local")) { + type = GST_WEBRTC_STATS_LOCAL_CANDIDATE; + } else if (strcmp (candidate_tag, "remote")) { + type = GST_WEBRTC_STATS_REMOTE_CANDIDATE; + } else { + GST_WARNING_OBJECT (webrtc, "Invalid ice candidate tag: %s", candidate_tag); + return NULL; + } + _set_base_stats (stats, type, ts, id); + + /* RTCIceCandidateStats + DOMString transportId; + DOMString address; + long port; + DOMString protocol; + RTCIceCandidateType candidateType; + long priority; + DOMString url; + DOMString relayProtocol; + */ + + if (transport_id) + gst_structure_set (stats, "transport-id", G_TYPE_STRING, transport_id, + NULL); + gst_structure_set (stats, "address", G_TYPE_STRING, can->ipaddr, NULL); + gst_structure_set (stats, "port", G_TYPE_UINT, can->port, NULL); + gst_structure_set (stats, "candidate-type", G_TYPE_STRING, can->type, NULL); + gst_structure_set (stats, "priority", G_TYPE_UINT, can->prio, NULL); + gst_structure_set (stats, "protocol", G_TYPE_STRING, can->proto, NULL); + if (can->relay_proto) + gst_structure_set (stats, "relay-protocol", G_TYPE_STRING, can->relay_proto, + NULL); + if (can->url) + gst_structure_set (stats, "url", G_TYPE_STRING, can->url, NULL); + + gst_structure_set (s, id, GST_TYPE_STRUCTURE, stats, NULL); + gst_structure_free (stats); + + return id; +} + +/* https://www.w3.org/TR/webrtc-stats/#candidatepair-dict* */ +static gchar * +_get_stats_from_ice_transport (GstWebRTCBin * webrtc, + GstWebRTCICETransport * transport, GstWebRTCICEStream * stream, + const GstStructure * twcc_stats, const gchar * transport_id, + GstStructure * s) +{ + GstStructure *stats; + gchar *id; + gchar *local_cand_id = NULL, *remote_cand_id = NULL; + double ts; + GstWebRTCICECandidateStats *local_cand = NULL, *remote_cand = NULL; + + gst_structure_get_double (s, "timestamp", &ts); + id = g_strdup_printf ("ice-candidate-pair_%s", GST_OBJECT_NAME (transport)); stats = gst_structure_new_empty (id); - _set_base_stats (stats, GST_WEBRTC_STATS_TRANSPORT, ts, id); + _set_base_stats (stats, GST_WEBRTC_STATS_CANDIDATE_PAIR, ts, id); -/* XXX: RTCIceCandidatePairStats - DOMString transportId; - DOMString localCandidateId; - DOMString remoteCandidateId; - RTCStatsIceCandidatePairState state; - unsigned long long priority; - boolean nominated; - unsigned long packetsSent; - unsigned long packetsReceived; - unsigned long long bytesSent; - unsigned long long bytesReceived; - DOMHighResTimeStamp lastPacketSentTimestamp; - DOMHighResTimeStamp lastPacketReceivedTimestamp; - DOMHighResTimeStamp firstRequestTimestamp; - DOMHighResTimeStamp lastRequestTimestamp; - DOMHighResTimeStamp lastResponseTimestamp; - double totalRoundTripTime; - double currentRoundTripTime; - double availableOutgoingBitrate; - double availableIncomingBitrate; - unsigned long circuitBreakerTriggerCount; - unsigned long long requestsReceived; - unsigned long long requestsSent; - unsigned long long responsesReceived; - unsigned long long responsesSent; - unsigned long long retransmissionsReceived; - unsigned long long retransmissionsSent; - unsigned long long consentRequestsSent; - DOMHighResTimeStamp consentExpiredTimestamp; -*/ + /* RTCIceCandidatePairStats + DOMString transportId; + DOMString localCandidateId; + DOMString remoteCandidateId; -/* XXX: RTCIceCandidateStats - DOMString transportId; - boolean isRemote; - RTCNetworkType networkType; - DOMString ip; - long port; - DOMString protocol; - RTCIceCandidateType candidateType; - long priority; - DOMString url; - DOMString relayProtocol; - boolean deleted = false; -}; -*/ + XXX: To be added: + + RTCStatsIceCandidatePairState state; + boolean nominated; + unsigned long packetsSent; + unsigned long packetsReceived; + unsigned long long bytesSent; + unsigned long long bytesReceived; + DOMHighResTimeStamp lastPacketSentTimestamp; + DOMHighResTimeStamp lastPacketReceivedTimestamp; + DOMHighResTimeStamp firstRequestTimestamp; + DOMHighResTimeStamp lastRequestTimestamp; + DOMHighResTimeStamp lastResponseTimestamp; + double totalRoundTripTime; + double currentRoundTripTime; + double availableOutgoingBitrate; + double availableIncomingBitrate; + unsigned long circuitBreakerTriggerCount; + unsigned long long requestsReceived; + unsigned long long requestsSent; + unsigned long long responsesReceived; + unsigned long long responsesSent; + unsigned long long retransmissionsReceived; + unsigned long long retransmissionsSent; + unsigned long long consentRequestsSent; + DOMHighResTimeStamp consentExpiredTimestamp; + unsigned long packetsDiscardedOnSend; + unsigned long long bytesDiscardedOnSend; + unsigned long long requestBytesSent; + unsigned long long consentRequestBytesSent; + unsigned long long responseBytesSent; + */ + + if (gst_webrtc_ice_get_selected_pair (webrtc->priv->ice, stream, + &local_cand, &remote_cand)) { + local_cand_id = + _get_stats_from_ice_candidates (webrtc, local_cand, transport_id, + "local", s); + remote_cand_id = + _get_stats_from_ice_candidates (webrtc, remote_cand, transport_id, + "remote", s); + + gst_structure_set (stats, "local-candidate-id", G_TYPE_STRING, + local_cand_id, NULL); + gst_structure_set (stats, "remote-candidate-id", G_TYPE_STRING, + remote_cand_id, NULL); + } else + GST_INFO_OBJECT (webrtc, + "No selected ICE candidate pair was found for transport %s", + GST_OBJECT_NAME (transport)); /* XXX: these stats are at the rtp session level but there isn't a specific * stats structure for that. The RTCIceCandidatePairStats is the closest with @@ -635,6 +707,20 @@ _get_stats_from_ice_transport (GstWebRTCBin * webrtc, NULL); gst_structure_set (s, id, GST_TYPE_STRUCTURE, stats, NULL); + + g_free (local_cand_id); + g_free (remote_cand_id); + + if (local_cand) { + g_free (local_cand->ipaddr); + g_free (local_cand->url); + } + if (remote_cand) + g_free (remote_cand->ipaddr); + + g_free (local_cand); + g_free (remote_cand); + gst_structure_free (stats); return id; @@ -643,8 +729,8 @@ _get_stats_from_ice_transport (GstWebRTCBin * webrtc, /* https://www.w3.org/TR/webrtc-stats/#dom-rtctransportstats */ static gchar * _get_stats_from_dtls_transport (GstWebRTCBin * webrtc, - GstWebRTCDTLSTransport * transport, const GstStructure * twcc_stats, - GstStructure * s) + GstWebRTCDTLSTransport * transport, GstWebRTCICEStream * stream, + const GstStructure * twcc_stats, GstStructure * s) { GstStructure *stats; gchar *id; @@ -677,26 +763,18 @@ _get_stats_from_dtls_transport (GstWebRTCBin * webrtc, DOMString issuerCertificateId; */ -/* XXX: RTCIceCandidateStats - DOMString transportId; - boolean isRemote; - DOMString ip; - long port; - DOMString protocol; - RTCIceCandidateType candidateType; - long priority; - DOMString url; - boolean deleted = false; -*/ + ice_id = + _get_stats_from_ice_transport (webrtc, transport->transport, stream, + twcc_stats, id, s); + if (ice_id) { + gst_structure_set (stats, "selected-candidate-pair-id", G_TYPE_STRING, + ice_id, NULL); + g_free (ice_id); + } gst_structure_set (s, id, GST_TYPE_STRUCTURE, stats, NULL); gst_structure_free (stats); - ice_id = - _get_stats_from_ice_transport (webrtc, transport->transport, twcc_stats, - s); - g_free (ice_id); - return id; } @@ -879,7 +957,7 @@ _get_stats_from_pad (GstWebRTCBin * webrtc, GstPad * pad, GstStructure * s) ts_stats.transport_id = _get_stats_from_dtls_transport (webrtc, ts_stats.stream->transport, - twcc_stats, s); + GST_WEBRTC_ICE_STREAM (ts_stats.stream->stream), twcc_stats, s); GST_DEBUG_OBJECT (webrtc, "retrieving rtp stream stats from transport %" GST_PTR_FORMAT " rtp session %" GST_PTR_FORMAT " with %u rtp sources, " diff --git a/subprojects/gst-plugins-bad/tests/check/elements/webrtcbin.c b/subprojects/gst-plugins-bad/tests/check/elements/webrtcbin.c index c6b853d187..e518a3b7f9 100644 --- a/subprojects/gst-plugins-bad/tests/check/elements/webrtcbin.c +++ b/subprojects/gst-plugins-bad/tests/check/elements/webrtcbin.c @@ -1559,6 +1559,29 @@ validate_remote_outbound_rtp_stats (const GstStructure * s, g_free (local_id); } +static void +validate_candidate_stats (const GstStructure * s, const GstStructure * stats) +{ + guint port; + guint64 priority; + gchar *address, *candidateType, *protocol; + + fail_unless (gst_structure_get (s, "address", G_TYPE_STRING, &address, NULL)); + fail_unless (gst_structure_get (s, "port", G_TYPE_UINT, &port, NULL)); + fail_unless (gst_structure_get (s, "candidate-type", G_TYPE_STRING, + &candidateType, NULL)); + fail_unless (gst_structure_get (s, "priority", G_TYPE_UINT64, &priority, + NULL)); + fail_unless (gst_structure_get (s, "protocol", G_TYPE_STRING, &protocol, + NULL)); + + fail_unless (strcmp (protocol, "udp") || strcmp (protocol, "tcp")); + + g_free (address); + g_free (candidateType); + g_free (protocol); +} + static gboolean validate_stats_foreach (GQuark field_id, const GValue * value, const GstStructure * stats) @@ -1592,7 +1615,9 @@ validate_stats_foreach (GQuark field_id, const GValue * value, } else if (type == GST_WEBRTC_STATS_TRANSPORT) { } else if (type == GST_WEBRTC_STATS_CANDIDATE_PAIR) { } else if (type == GST_WEBRTC_STATS_LOCAL_CANDIDATE) { + validate_candidate_stats (s, stats); } else if (type == GST_WEBRTC_STATS_REMOTE_CANDIDATE) { + validate_candidate_stats (s, stats); } else if (type == GST_WEBRTC_STATS_CERTIFICATE) { } else { g_assert_not_reached (); @@ -1646,6 +1671,53 @@ GST_START_TEST (test_session_stats) GST_END_TEST; +GST_START_TEST (test_stats_with_stream) +{ + struct test_webrtc *t = create_audio_test (); + GstPromise *p; + GstCaps *caps; + GstPad *pad; + + /* test that the stats generated with stream are sane */ + + t->on_offer_created = NULL; + t->on_answer_created = NULL; + t->on_negotiation_needed = NULL; + + fail_if (gst_element_set_state (t->webrtc1, + GST_STATE_READY) == GST_STATE_CHANGE_FAILURE); + fail_if (gst_element_set_state (t->webrtc2, + GST_STATE_READY) == GST_STATE_CHANGE_FAILURE); + + test_webrtc_create_offer (t); + + fail_if (gst_element_set_state (t->webrtc1, + GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE); + fail_if (gst_element_set_state (t->webrtc2, + GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE); + + /* set caps for webrtcbin sink to validate codec stats */ + caps = gst_caps_from_string (OPUS_RTP_CAPS (96)); + pad = gst_element_get_static_pad (t->webrtc1, "sink_0"); + gst_pad_set_caps (pad, caps); + gst_caps_unref (caps); + + test_webrtc_wait_for_answer_error_eos (t); + fail_unless (t->state == STATE_ANSWER_SET); + + p = gst_promise_new_with_change_func (_on_stats, t, NULL); + g_signal_emit_by_name (t->webrtc1, "get-stats", NULL, p); + p = gst_promise_new_with_change_func (_on_stats, t, NULL); + g_signal_emit_by_name (t->webrtc2, "get-stats", NULL, p); + + test_webrtc_wait_for_state_mask (t, 1 << STATE_CUSTOM); + + gst_object_unref (pad); + test_webrtc_free (t); +} + +GST_END_TEST; + GST_START_TEST (test_add_transceiver) { struct test_webrtc *t = test_webrtc_new (); @@ -5260,6 +5332,7 @@ webrtcbin_suite (void) if (nicesrc && nicesink && dtlssrtpenc && dtlssrtpdec) { tcase_add_test (tc, test_sdp_no_media); tcase_add_test (tc, test_session_stats); + tcase_add_test (tc, test_stats_with_stream); tcase_add_test (tc, test_audio); tcase_add_test (tc, test_ice_port_restriction); tcase_add_test (tc, test_audio_video);