From 138d00f266c0d6071f5709ed4b352a7d505fa1a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=C5=9Aled=C5=BA?= Date: Tue, 17 Oct 2023 18:49:49 +0200 Subject: [PATCH 1/2] Create transceivers on set_remote_description --- examples/example.exs | 1 - lib/ex_webrtc/media_stream_track.ex | 23 +++++ lib/ex_webrtc/peer_connection.ex | 55 +++++++++-- .../peer_connection/configuration.ex | 2 +- lib/ex_webrtc/rtp_transceiver.ex | 48 +++++++++ lib/ex_webrtc/utils.ex | 6 ++ test/peer_connection_test.exs | 98 +++++++++++++++++++ 7 files changed, 224 insertions(+), 9 deletions(-) create mode 100644 lib/ex_webrtc/media_stream_track.ex create mode 100644 lib/ex_webrtc/rtp_transceiver.ex create mode 100644 test/peer_connection_test.exs diff --git a/examples/example.exs b/examples/example.exs index 4000e310..02fed4e5 100644 --- a/examples/example.exs +++ b/examples/example.exs @@ -31,7 +31,6 @@ defmodule Peer do Process.send_after(self(), :ws_ping, 1000) {:ok, pc} = PeerConnection.start_link( - bundle_policy: :max_bundle, ice_servers: @ice_servers ) diff --git a/lib/ex_webrtc/media_stream_track.ex b/lib/ex_webrtc/media_stream_track.ex new file mode 100644 index 00000000..d68cf249 --- /dev/null +++ b/lib/ex_webrtc/media_stream_track.ex @@ -0,0 +1,23 @@ +defmodule ExWebRTC.MediaStreamTrack do + @moduledoc """ + MediaStreamTrack + """ + + @type t() :: %__MODULE__{ + kind: :audio | :video, + id: integer(), + mid: String.t() + } + + @enforce_keys [:id, :kind] + defstruct @enforce_keys ++ [:mid] + + def from_transceiver(tr) do + %__MODULE__{kind: tr.kind, id: generate_id(), mid: tr.mid} + end + + defp generate_id() do + <> = :crypto.strong_rand_bytes(12) + id + end +end diff --git a/lib/ex_webrtc/peer_connection.ex b/lib/ex_webrtc/peer_connection.ex index 6564083e..1c6c6f7b 100644 --- a/lib/ex_webrtc/peer_connection.ex +++ b/lib/ex_webrtc/peer_connection.ex @@ -7,7 +7,7 @@ defmodule ExWebRTC.PeerConnection do alias __MODULE__.Configuration alias ExICE.ICEAgent - alias ExWebRTC.{IceCandidate, SessionDescription} + alias ExWebRTC.{IceCandidate, MediaStreamTrack, RTPTransceiver, SessionDescription} import ExWebRTC.Utils @@ -108,6 +108,11 @@ defmodule ExWebRTC.PeerConnection do GenServer.call(peer_connection, {:add_ice_candidate, candidate}) end + @spec get_transceivers(peer_connection()) :: [RTPTransceiver.t()] + def get_transceivers(peer_connection) do + GenServer.call(peer_connection, :get_transceivers) + end + #### CALLBACKS #### @impl true @@ -213,6 +218,10 @@ defmodule ExWebRTC.PeerConnection do {:reply, :ok, state} end + def handle_call(:get_transceivers, _from, state) do + {:reply, state.transceivers, state} + end + @impl true def handle_info({:ex_ice, _from, :connected}, state) do if state.dtls_buffered_packets do @@ -232,7 +241,7 @@ defmodule ExWebRTC.PeerConnection do # username_fragment: "vx/1" } - send(state.owner, {:ex_webrtc, {:ice_candidate, candidate}}) + notify(state.owner, {:ice_candidate, candidate}) {:noreply, state} end @@ -293,13 +302,43 @@ defmodule ExWebRTC.PeerConnection do defp apply_remote_description(_type, sdp, state) do # TODO apply steps listed in RFC 8829 5.10 media = hd(sdp.media) - {:ice_ufrag, ufrag} = ExSDP.Media.get_attribute(media, :ice_ufrag) - {:ice_pwd, pwd} = ExSDP.Media.get_attribute(media, :ice_pwd) - :ok = ICEAgent.set_remote_credentials(state.ice_agent, ufrag, pwd) - :ok = ICEAgent.gather_candidates(state.ice_agent) + with {:ice_ufrag, ufrag} <- ExSDP.Media.get_attribute(media, :ice_ufrag), + {:ice_pwd, pwd} <- ExSDP.Media.get_attribute(media, :ice_pwd), + {:ok, new_transceivers} <- update_transceivers(state.transceivers, sdp) do + :ok = ICEAgent.set_remote_credentials(state.ice_agent, ufrag, pwd) + :ok = ICEAgent.gather_candidates(state.ice_agent) + + new_remote_tracks = + new_transceivers + # only take new transceivers that can receive tracks + |> Enum.filter(fn tr -> + RTPTransceiver.find_by_mid(state.transceivers, tr.mid) == nil and + tr.direction in [:recvonly, :sendrecv] + end) + |> Enum.map(fn tr -> MediaStreamTrack.from_transceiver(tr) end) + + for track <- new_remote_tracks do + notify(state.owner, {:track, track}) + end + + {:ok, %{state | current_remote_desc: sdp, transceivers: new_transceivers}} + else + nil -> {:error, :missing_ice_ufrag_or_pwd} + end + end - {:ok, %{state | current_remote_desc: sdp}} + defp update_transceivers(transceivers, sdp) do + Enum.reduce_while(sdp.media, {:ok, transceivers}, fn mline, {:ok, transceivers} -> + case ExSDP.Media.get_attribute(mline, :mid) do + {:mid, mid} -> + transceivers = RTPTransceiver.update_or_create(transceivers, mid, mline) + {:cont, {:ok, transceivers}} + + _other -> + {:halt, {:error, :missing_mid}} + end + end) end # Signaling state machine, RFC 8829 3.2 @@ -326,4 +365,6 @@ defmodule ExWebRTC.PeerConnection do defp maybe_next_state(:have_remote_pranswer, :remote, :answer), do: {:ok, :stable} defp maybe_next_state(:have_remote_pranswer, _, _), do: {:error, :invalid_transition} + + defp notify(pid, msg), do: send(pid, {:ex_webrtc, self(), msg}) end diff --git a/lib/ex_webrtc/peer_connection/configuration.ex b/lib/ex_webrtc/peer_connection/configuration.ex index 34fdc038..c197389e 100644 --- a/lib/ex_webrtc/peer_connection/configuration.ex +++ b/lib/ex_webrtc/peer_connection/configuration.ex @@ -33,7 +33,7 @@ defmodule ExWebRTC.PeerConnection.Configuration do rtcp_mux_policy: rtcp_mux_policy() } - defstruct bundle_policy: :balanced, + defstruct bundle_policy: :max_bundle, certificates: nil, ice_candidate_pool_size: 0, ice_servers: [], diff --git a/lib/ex_webrtc/rtp_transceiver.ex b/lib/ex_webrtc/rtp_transceiver.ex new file mode 100644 index 00000000..238c6cce --- /dev/null +++ b/lib/ex_webrtc/rtp_transceiver.ex @@ -0,0 +1,48 @@ +defmodule ExWebRTC.RTPTransceiver do + @moduledoc """ + RTPTransceiver + """ + + @type t() :: %__MODULE__{ + mid: String.t(), + direction: :sendonly | :recvonly | :sendrecv | :inactive | :stopped, + kind: :audio | :video + } + + @enforce_keys [:mid, :direction, :kind] + defstruct @enforce_keys + + @doc false + def find_by_mid(transceivers, mid) do + transceivers + |> Enum.with_index(fn tr, idx -> {idx, tr} end) + |> Enum.find(fn {_idx, tr} -> tr.mid == mid end) + end + + # searches for transceiver for a given mline + # if it exists, updates its configuration + # if it doesn't exist, creats a new one + # returns list of updated transceivers + @doc false + def update_or_create(transceivers, mid, mline) do + case find_by_mid(transceivers, mid) do + {idx, %__MODULE__{} = tr} -> + case update(tr, mline) do + {:ok, tr} -> List.replace_at(transceivers, idx, tr) + {:error, :remove} -> List.delete_at(transceivers, idx) + end + + nil -> + transceivers ++ [%__MODULE__{mid: mid, direction: :recvonly, kind: mline.type}] + end + end + + defp update(transceiver, mline) do + # if there is no direction, the default is sendrecv + # see RFC 3264, sec. 6.1 + case ExWebRTC.Utils.get_media_direction(mline) || :sendrecv do + :inactive -> {:error, :remove} + other_direction -> {:ok, %__MODULE__{transceiver | direction: other_direction}} + end + end +end diff --git a/lib/ex_webrtc/utils.ex b/lib/ex_webrtc/utils.ex index 2d6d9b43..0581524f 100644 --- a/lib/ex_webrtc/utils.ex +++ b/lib/ex_webrtc/utils.ex @@ -7,4 +7,10 @@ defmodule ExWebRTC.Utils do |> :binary.bin_to_list() |> Enum.map_join(":", &Base.encode16(<<&1>>)) end + + def get_media_direction(media) do + Enum.find(media.attributes, fn attr -> + attr in [:sendrecv, :sendonly, :recvonly, :inactive] + end) + end end diff --git a/test/peer_connection_test.exs b/test/peer_connection_test.exs new file mode 100644 index 00000000..2c424b0a --- /dev/null +++ b/test/peer_connection_test.exs @@ -0,0 +1,98 @@ +defmodule ExWebRTC.PeerConnectionTest do + use ExUnit.Case, async: true + + alias ExWebRTC.{MediaStreamTrack, PeerConnection, SessionDescription} + + @single_audio_offer """ + v=0 + o=- 6788894006044524728 2 IN IP4 127.0.0.1 + s=- + t=0 0 + a=group:BUNDLE 0 + a=extmap-allow-mixed + a=msid-semantic: WMS + m=audio 9 UDP/TLS/RTP/SAVPF 111 + c=IN IP4 0.0.0.0 + a=rtcp:9 IN IP4 0.0.0.0 + a=ice-ufrag:cDua + a=ice-pwd:v9SCmZHxJWtgpyzn8Ts1puT6 + a=ice-options:trickle + a=fingerprint:sha-256 11:35:68:66:A4:C3:C0:AA:37:4E:0F:97:D7:9F:76:11:08:DB:56:DA:4B:83:77:50:9A:D2:71:8D:2A:A8:E3:07 + a=setup:actpass + a=mid:0 + a=sendrecv + a=msid:- 54f0751b-086f-433c-af40-79c179182423 + a=rtcp-mux + a=rtpmap:111 opus/48000/2 + a=rtcp-fb:111 transport-cc + a=fmtp:111 minptime=10;useinbandfec=1 + a=ssrc:1463342914 cname:poWwjNZ4I2ZZgzY7 + a=ssrc:1463342914 msid:- 54f0751b-086f-433c-af40-79c179182423 + """ + + @audio_video_offer """ + v=0 + o=- 3253533641493747086 5 IN IP4 127.0.0.1 + s=- + t=0 0 + a=group:BUNDLE 0 1 + a=extmap-allow-mixed + a=msid-semantic: WMS + m=audio 9 UDP/TLS/RTP/SAVPF 111 + c=IN IP4 0.0.0.0 + a=rtcp:9 IN IP4 0.0.0.0 + a=ice-ufrag:SOct + a=ice-pwd:k9PRXt7zT32ADt/juUpt4Gx3 + a=ice-options:trickle + a=fingerprint:sha-256 45:B5:2D:3A:DA:29:93:27:B6:59:F1:5B:77:62:F5:C2:CE:16:8B:12:C7:B8:34:EF:C0:12:45:17:D0:1A:E6:F4 + a=setup:actpass + a=mid:0 + a=sendrecv + a=msid:- 0970fb0b-4750-4302-902e-70d2e403ad0d + a=rtcp-mux + a=rtpmap:111 opus/48000/2 + a=rtcp-fb:111 transport-cc + a=fmtp:111 minptime=10;useinbandfec=1 + a=ssrc:560549895 cname:QQJypppcjR+gR484 + a=ssrc:560549895 msid:- 0970fb0b-4750-4302-902e-70d2e403ad0d + m=video 9 UDP/TLS/RTP/SAVPF 96 + c=IN IP4 0.0.0.0 + a=rtcp:9 IN IP4 0.0.0.0 + a=ice-ufrag:SOct + a=ice-pwd:k9PRXt7zT32ADt/juUpt4Gx3 + a=ice-options:trickle + a=fingerprint:sha-256 45:B5:2D:3A:DA:29:93:27:B6:59:F1:5B:77:62:F5:C2:CE:16:8B:12:C7:B8:34:EF:C0:12:45:17:D0:1A:E6:F4 + a=setup:actpass + a=mid:1 + a=sendrecv + a=msid:- 1259ea70-c6b7-445a-9c20-49cec7433ccb + a=rtcp-mux + a=rtcp-rsize + a=rtpmap:96 VP8/90000 + a=rtcp-fb:96 goog-remb + a=rtcp-fb:96 transport-cc + a=rtcp-fb:96 ccm fir + a=rtcp-fb:96 nack + a=rtcp-fb:96 nack pli + a=ssrc-group:FID 381060598 184440407 + a=ssrc:381060598 cname:QQJypppcjR+gR484 + a=ssrc:381060598 msid:- 1259ea70-c6b7-445a-9c20-49cec7433ccb + a=ssrc:184440407 cname:QQJypppcjR+gR484 + a=ssrc:184440407 msid:- 1259ea70-c6b7-445a-9c20-49cec7433ccb + """ + + test "transceivers" do + {:ok, pc} = PeerConnection.start_link() + + offer = %SessionDescription{type: :offer, sdp: @single_audio_offer} + :ok = PeerConnection.set_remote_description(pc, offer) + + assert_receive {:ex_webrtc, ^pc, {:track, %MediaStreamTrack{mid: "0", kind: :audio}}} + + offer = %SessionDescription{type: :offer, sdp: @audio_video_offer} + :ok = PeerConnection.set_remote_description(pc, offer) + + assert_receive {:ex_webrtc, ^pc, {:track, %MediaStreamTrack{mid: "1", kind: :video}}} + refute_receive {:ex_webrtc, ^pc, {:track, %MediaStreamTrack{}}} + end +end From f89f40a5ff4771bb20220bd23c5166ffa1642d16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=C5=9Aled=C5=BA?= Date: Thu, 19 Oct 2023 20:19:57 +0200 Subject: [PATCH 2/2] Don't remove transceivers when they become inactive --- lib/ex_webrtc/rtp_transceiver.ex | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/lib/ex_webrtc/rtp_transceiver.ex b/lib/ex_webrtc/rtp_transceiver.ex index 238c6cce..bf5d0375 100644 --- a/lib/ex_webrtc/rtp_transceiver.ex +++ b/lib/ex_webrtc/rtp_transceiver.ex @@ -27,22 +27,12 @@ defmodule ExWebRTC.RTPTransceiver do def update_or_create(transceivers, mid, mline) do case find_by_mid(transceivers, mid) do {idx, %__MODULE__{} = tr} -> - case update(tr, mline) do - {:ok, tr} -> List.replace_at(transceivers, idx, tr) - {:error, :remove} -> List.delete_at(transceivers, idx) - end + List.replace_at(transceivers, idx, update(tr, mline)) nil -> transceivers ++ [%__MODULE__{mid: mid, direction: :recvonly, kind: mline.type}] end end - defp update(transceiver, mline) do - # if there is no direction, the default is sendrecv - # see RFC 3264, sec. 6.1 - case ExWebRTC.Utils.get_media_direction(mline) || :sendrecv do - :inactive -> {:error, :remove} - other_direction -> {:ok, %__MODULE__{transceiver | direction: other_direction}} - end - end + defp update(transceiver, _mline), do: transceiver end