From 31c1a79acf2f49ba5cf9171b456f2688d86b8ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=C5=9Aled=C5=BA?= Date: Fri, 20 Oct 2023 17:41:53 +0200 Subject: [PATCH] Generate answer from RTPTransceivers --- examples/example.exs | 6 +- lib/ex_webrtc/peer_connection.ex | 95 ++++++++++----------------- lib/ex_webrtc/rtp_codec_parameters.ex | 18 +++++ lib/ex_webrtc/rtp_receiver.ex | 3 + lib/ex_webrtc/rtp_transceiver.ex | 83 +++++++++++++++++++++-- test/peer_connection_test.exs | 3 + 6 files changed, 142 insertions(+), 66 deletions(-) create mode 100644 lib/ex_webrtc/rtp_codec_parameters.ex create mode 100644 lib/ex_webrtc/rtp_receiver.ex diff --git a/examples/example.exs b/examples/example.exs index 02fed4e5..ceb4643e 100644 --- a/examples/example.exs +++ b/examples/example.exs @@ -71,7 +71,7 @@ defmodule Peer do end @impl true - def handle_info({:ex_webrtc, msg}, state) do + def handle_info({:ex_webrtc, _pid, msg}, state) do Logger.info("Received ExWebRTC message: #{inspect(msg)}") handle_webrtc_message(msg, state) @@ -119,6 +119,10 @@ defmodule Peer do :gun.ws_send(state.conn, state.stream, {:text, Jason.encode!(msg)}) end + + defp handle_webrtc_message(msg, _state) do + Logger.warning("Received unknown ex_webrtc message: #{inspect(msg)}") + end end {:ok, pid} = Peer.start_link() diff --git a/lib/ex_webrtc/peer_connection.ex b/lib/ex_webrtc/peer_connection.ex index 1c6c6f7b..25d97e8a 100644 --- a/lib/ex_webrtc/peer_connection.ex +++ b/lib/ex_webrtc/peer_connection.ex @@ -32,42 +32,6 @@ defmodule ExWebRTC.PeerConnection do signaling_state: :stable ] - @dummy_sdp """ - v=0 - o=- 7596991810024734139 2 IN IP4 127.0.0.1 - s=- - t=0 0 - a=group:BUNDLE 0 - a=extmap-allow-mixed - a=msid-semantic: WMS - m=audio 9 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126 - c=IN IP4 0.0.0.0 - a=rtcp:9 IN IP4 0.0.0.0 - a=ice-ufrag:vx/1 - a=ice-pwd:ldFUrCsXvndFY2L1u0UQ7ikf - a=ice-options:trickle - a=fingerprint:sha-256 76:61:77:1E:7C:2E:BB:CD:19:B5:27:4E:A7:40:84:06:6B:17:97:AB:C4:61:90:16:EE:96:9F:9E:BD:42:96:3D - a=setup:passive - a=mid:0 - a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level - a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time - a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 - a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid - a=recvonly - a=rtcp-mux - a=rtpmap:111 opus/48000/2 - a=rtcp-fb:111 transport-cc - a=fmtp:111 minptime=10;useinbandfec=1 - a=rtpmap:63 red/48000/2 - a=fmtp:63 111/111 - a=rtpmap:9 G722/8000 - a=rtpmap:0 PCMU/8000 - a=rtpmap:8 PCMA/8000 - a=rtpmap:13 CN/8000 - a=rtpmap:110 telephone-event/48000 - a=rtpmap:126 telephone-event/8000 - """ - #### API #### def start_link(configuration \\ []) do @@ -147,36 +111,45 @@ defmodule ExWebRTC.PeerConnection do @impl true def handle_call({:create_answer, _options}, _from, state) when state.signaling_state in [:have_remote_offer, :have_local_pranswer] do - {:ok, ufrag, pwd} = ICEAgent.get_local_credentials(state.ice_agent) - + {:ok, ice_ufrag, ice_pwd} = ICEAgent.get_local_credentials(state.ice_agent) {:ok, dtls_fingerprint} = ExDTLS.get_cert_fingerprint(state.dtls_client) - sdp = ExSDP.parse!(@dummy_sdp) - media = hd(sdp.media) - - attrs = - Enum.map(media.attributes, fn - {:ice_ufrag, _} -> - {:ice_ufrag, ufrag} - - {:ice_pwd, _} -> - {:ice_pwd, pwd} - - {:fingerprint, {hash_function, _}} -> - {:fingerprint, {hash_function, hex_dump(dtls_fingerprint)}} - - other -> - other + answer = %ExSDP{ExSDP.new() | timing: %ExSDP.Timing{start_time: 0, stop_time: 0}} + + config = + [ + ice_ufrag: ice_ufrag, + ice_pwd: ice_pwd, + ice_options: "trickle", + fingerprint: {:sha256, hex_dump(dtls_fingerprint)}, + # TODO offer will always contain actpass + # and answer should contain active + # see RFC 8829 sec. 5.3.1 + setup: :passive + ] + + mlines = + Enum.map(state.transceivers, fn transceiver -> + RTPTransceiver.to_mline(transceiver, config) end) - media = Map.put(media, :attributes, attrs) - - sdp = - sdp - |> Map.put(:media, [media]) - |> to_string() + mids = + Enum.map(mlines, fn mline -> + {:mid, mid} = ExSDP.Media.get_attribute(mline, :mid) + mid + end) - desc = %SessionDescription{type: :answer, sdp: sdp} + answer = + answer + |> ExSDP.add_attributes([ + %ExSDP.Attribute.Group{semantics: "BUNDLE", mids: mids}, + # always allow for mixing one- and two-byte RTP header extensions + # TODO ensure this was also offered + "extmap-allow-mixed" + ]) + |> ExSDP.add_media(mlines) + + desc = %SessionDescription{type: :answer, sdp: to_string(answer)} {:reply, {:ok, desc}, state} end diff --git a/lib/ex_webrtc/rtp_codec_parameters.ex b/lib/ex_webrtc/rtp_codec_parameters.ex new file mode 100644 index 00000000..d240f6b2 --- /dev/null +++ b/lib/ex_webrtc/rtp_codec_parameters.ex @@ -0,0 +1,18 @@ +defmodule ExWebRTC.RTPCodecParameters do + @moduledoc """ + RTPCodecParameters + """ + + defstruct [:payload_type, :mime_type, :clock_rate, :channels, :sdp_fmtp_line, :rtcp_fbs] + + def new(type, rtp_mapping, fmtp, rtcp_fbs) do + %__MODULE__{ + payload_type: rtp_mapping.payload_type, + mime_type: "#{type}/#{rtp_mapping.encoding}", + clock_rate: rtp_mapping.clock_rate, + channels: rtp_mapping.params, + sdp_fmtp_line: fmtp, + rtcp_fbs: rtcp_fbs + } + end +end diff --git a/lib/ex_webrtc/rtp_receiver.ex b/lib/ex_webrtc/rtp_receiver.ex new file mode 100644 index 00000000..19237a18 --- /dev/null +++ b/lib/ex_webrtc/rtp_receiver.ex @@ -0,0 +1,3 @@ +defmodule ExWebRTC.RTPReceiver do + defstruct [:ssrc] +end diff --git a/lib/ex_webrtc/rtp_transceiver.ex b/lib/ex_webrtc/rtp_transceiver.ex index bf5d0375..5ad1d49b 100644 --- a/lib/ex_webrtc/rtp_transceiver.ex +++ b/lib/ex_webrtc/rtp_transceiver.ex @@ -3,14 +3,19 @@ defmodule ExWebRTC.RTPTransceiver do RTPTransceiver """ + alias ExWebRTC.{RTPCodecParameters, RTPReceiver} + @type t() :: %__MODULE__{ mid: String.t(), direction: :sendonly | :recvonly | :sendrecv | :inactive | :stopped, - kind: :audio | :video + kind: :audio | :video, + hdr_exts: [], + codecs: [], + rtp_receiver: nil } @enforce_keys [:mid, :direction, :kind] - defstruct @enforce_keys + defstruct @enforce_keys ++ [codecs: [], hdr_exts: [], rtp_receiver: %RTPReceiver{}] @doc false def find_by_mid(transceivers, mid) do @@ -30,9 +35,79 @@ defmodule ExWebRTC.RTPTransceiver do List.replace_at(transceivers, idx, update(tr, mline)) nil -> - transceivers ++ [%__MODULE__{mid: mid, direction: :recvonly, kind: mline.type}] + codecs = get_codecs(mline) + hdr_exts = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.Extmap) + ssrc = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.SSRC) + + tr = %__MODULE__{ + mid: mid, + direction: :recvonly, + kind: mline.type, + codecs: codecs, + hdr_exts: hdr_exts, + rtp_receiver: %RTPReceiver{ssrc: ssrc} + } + + transceivers ++ [tr] end end - defp update(transceiver, _mline), do: transceiver + def to_mline(transceiver, config) do + pt = Enum.map(transceiver.codecs, fn codec -> codec.payload_type end) + + media_formats = + Enum.flat_map(transceiver.codecs, fn codec -> + [_type, encoding] = String.split(codec.mime_type, "/") + + rtp_mapping = %ExSDP.Attribute.RTPMapping{ + clock_rate: codec.clock_rate, + encoding: encoding, + params: codec.channels, + payload_type: codec.payload_type + } + + [rtp_mapping, codec.sdp_fmtp_line, codec.rtcp_fbs] + end) + + %ExSDP.Media{ + ExSDP.Media.new(transceiver.kind, 9, "UDP/TLS/RTP/SAVPF", pt) + | # mline must be followed by a cline, which must contain + # the default value "IN IP4 0.0.0.0" (as there are no candidates yet) + connection_data: [%ExSDP.ConnectionData{address: {0, 0, 0, 0}}] + } + |> ExSDP.Media.add_attributes([ + transceiver.direction, + {:mid, transceiver.mid}, + {:ice_ufrag, Keyword.fetch!(config, :ice_ufrag)}, + {:ice_pwd, Keyword.fetch!(config, :ice_pwd)}, + {:ice_options, Keyword.fetch!(config, :ice_options)}, + {:fingerprint, Keyword.fetch!(config, :fingerprint)}, + {:setup, Keyword.fetch!(config, :setup)}, + :rtcp_mux + ]) + |> ExSDP.Media.add_attributes(media_formats) + end + + defp update(transceiver, mline) do + codecs = get_codecs(mline) + hdr_exts = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.Extmap) + ssrc = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.SSRC) + rtp_receiver = %RTPReceiver{ssrc: ssrc} + %__MODULE__{transceiver | codecs: codecs, hdr_exts: hdr_exts, rtp_receiver: rtp_receiver} + end + + defp get_codecs(mline) do + rtp_mappings = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.RTPMapping) + fmtps = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.FMTP) + all_rtcp_fbs = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.RTCPFeedback) + + for rtp_mapping <- rtp_mappings do + fmtp = Enum.find(fmtps, fn fmtp -> fmtp.pt == rtp_mapping.payload_type end) + + rtcp_fbs = + Enum.filter(all_rtcp_fbs, fn rtcp_fb -> rtcp_fb.pt == rtp_mapping.payload_type end) + + RTPCodecParameters.new(mline.type, rtp_mapping, fmtp, rtcp_fbs) + end + end end diff --git a/test/peer_connection_test.exs b/test/peer_connection_test.exs index 2c424b0a..0d5aef20 100644 --- a/test/peer_connection_test.exs +++ b/test/peer_connection_test.exs @@ -82,6 +82,7 @@ defmodule ExWebRTC.PeerConnectionTest do """ test "transceivers" do + IO.inspect(ExSDP.parse(@audio_video_offer)) {:ok, pc} = PeerConnection.start_link() offer = %SessionDescription{type: :offer, sdp: @single_audio_offer} @@ -92,6 +93,8 @@ defmodule ExWebRTC.PeerConnectionTest do offer = %SessionDescription{type: :offer, sdp: @audio_video_offer} :ok = PeerConnection.set_remote_description(pc, offer) + IO.inspect(PeerConnection.get_transceivers(pc)) + assert_receive {:ex_webrtc, ^pc, {:track, %MediaStreamTrack{mid: "1", kind: :video}}} refute_receive {:ex_webrtc, ^pc, {:track, %MediaStreamTrack{}}} end