From d72ffd982f97c2268ea012fbbdd84b12f440ff89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=C5=9Aled=C5=BA?= Date: Sat, 4 Nov 2023 10:08:06 +0100 Subject: [PATCH] wip --- lib/ex_webrtc/peer_connection.ex | 19 ++- .../peer_connection/configuration.ex | 108 ++++++++++++++++-- lib/ex_webrtc/rtp_transceiver.ex | 47 +++++--- 3 files changed, 135 insertions(+), 39 deletions(-) diff --git a/lib/ex_webrtc/peer_connection.ex b/lib/ex_webrtc/peer_connection.ex index c9e1f69a..4748d83b 100644 --- a/lib/ex_webrtc/peer_connection.ex +++ b/lib/ex_webrtc/peer_connection.ex @@ -95,13 +95,10 @@ defmodule ExWebRTC.PeerConnection do GenServer.call(peer_connection, :get_transceivers) end - @spec add_transceiver( - peer_connection(), - RTPTransceiver.kind() | MediaStreamTrack.t(), - transceiver_options() - ) :: {:ok, RTPTransceiver.t()} | {:error, :TODO} - def add_transceiver(peer_connection, track_or_kind, options \\ []) do - GenServer.call(peer_connection, {:add_transceiver, track_or_kind, options}) + @spec add_transceiver(peer_connection(), RTPTransceiver.kind(), transceiver_options()) :: + {:ok, RTPTransceiver.t()} | {:error, :TODO} + def add_transceiver(peer_connection, kind, options \\ []) do + GenServer.call(peer_connection, {:add_transceiver, kind, options}) end #### CALLBACKS #### @@ -300,7 +297,6 @@ defmodule ExWebRTC.PeerConnection do @impl true def handle_call({:add_transceiver, :audio, options}, _from, state) do - # TODO: proper implementation, change the :audio above to track_or_kind direction = Keyword.get(options, :direction, :sendrcv) # hardcoded audio codec @@ -386,7 +382,8 @@ defmodule ExWebRTC.PeerConnection do with :ok <- SDPUtils.ensure_mid(sdp), :ok <- SDPUtils.ensure_bundle(sdp), {:ok, {ice_ufrag, ice_pwd}} <- SDPUtils.get_ice_credentials(sdp), - {:ok, new_transceivers} <- update_remote_transceivers(state.transceivers, sdp) do + {:ok, new_transceivers} <- + update_remote_transceivers(state.transceivers, sdp, state.config) do :ok = ICEAgent.set_remote_credentials(state.ice_agent, ice_ufrag, ice_pwd) :ok = ICEAgent.gather_candidates(state.ice_agent) @@ -424,11 +421,11 @@ defmodule ExWebRTC.PeerConnection do end end - defp update_remote_transceivers(transceivers, sdp) do + defp update_remote_transceivers(transceivers, sdp, config) do Enum.reduce_while(sdp.media, {:ok, transceivers}, fn mline, {:ok, transceivers} -> case ExSDP.Media.get_attribute(mline, :mid) do {:mid, mid} -> - transceivers = RTPTransceiver.update_or_create(transceivers, mid, mline) + transceivers = RTPTransceiver.update_or_create(transceivers, mid, mline, config) {:cont, {:ok, transceivers}} _other -> diff --git a/lib/ex_webrtc/peer_connection/configuration.ex b/lib/ex_webrtc/peer_connection/configuration.ex index 865d8f73..9566540d 100644 --- a/lib/ex_webrtc/peer_connection/configuration.ex +++ b/lib/ex_webrtc/peer_connection/configuration.ex @@ -3,16 +3,47 @@ defmodule ExWebRTC.PeerConnection.Configuration do PeerConnection configuration """ + @default_codecs [:opus, :h264, :vp8] + + @codecs %{ + :audio => [%{name: :opus, clock_rate: 48_000, channels: 2, payload_type: 111}], + :video => [ + %{name: :h264, clock_rate: 90_000, channels: nil, payload_type: 98}, + %{name: :vp8, clock_rate: 90_000, channels: nil, payload_type: 96} + ] + } + + @rtp_hdr_extensions %{ + :mid => "urn:ietf:params:rtp-hdrext:sdes:mid", + :audio_level => "urn:ietf:params:rtp-hdrext:ssrc-audio-level" + } + + @mandatory_rtp_hdr_exts [:mid] + @type ice_server() :: %{ optional(:credential) => String.t(), optional(:username) => String.t(), :urls => [String.t()] | String.t() } + @type codec() :: :opus | :h264 | :vp8 + + @type rtp_hdr_extension() :: :audio_level + @typedoc """ Options that can be passed to `ExWebRTC.PeerConnection.start_link/1`. - Currently, ExWebRTC always uses the following config: + * `ice_servers` - list of STUN servers to use. + TURN servers are not supported right now and will be filtered out. + * `codecs` - list of codecs to use. + Use `default_codecs/0` to get a list of default codecs. + This option overrides default codecs. + If you wish to add codecs to default ones do + `codecs: Configuration.get_default_codecs() ++ my_codecs` + * `rtp_hdr_extensions` - list of RTP header extensions to use. + MID extension is enabled by default and cannot be turned off. + + Besides options listed above, ExWebRTC uses the following config: * bundle_policy - max_bundle * ice_candidate_pool_size - 0 * ice_transport_policy - all @@ -20,24 +51,81 @@ defmodule ExWebRTC.PeerConnection.Configuration do This config cannot be changed. """ - @type options() :: [ice_servers: [ice_server()]] + @type options() :: [ + ice_servers: [ice_server()], + codecs: [codec()], + rtp_hdr_extensions: [rtp_hdr_extension()] + ] @typedoc false - @type t() :: %__MODULE__{ice_servers: [ice_server()]} + @type t() :: %__MODULE__{ + ice_servers: [ice_server()], + codecs: [codec()], + rtp_hdr_extensions: [rtp_hdr_extension()] + } + + defstruct ice_servers: [], + codecs: @default_codecs, + rtp_hdr_extensions: @mandatory_rtp_hdr_exts - defstruct ice_servers: [] + @doc """ + Returns a list of codecs that are used by default. + """ + @spec default_codecs() :: [codec()] + def default_codecs(), do: @default_codecs @doc false @spec from_options!(options()) :: t() def from_options!(options) do - config = struct!(__MODULE__, options) + options = + options + |> add_mandatory_rtp_hdr_extensions() + |> resolve_rtp_hdr_extensions() + |> resolve_codecs() + # ATM, ExICE does not support relay via TURN + |> reject_turn_servers() + + struct!(__MODULE__, options) + end + + @doc false + @spec is_supported_codec(t(), ExSDP.Attribute.RTPMapping.t()) :: boolean() + def is_supported_codec(config, rtp_mapping) do + supported_codecs = Enum.map(config.codecs, fn codec -> "#{codec}" end) + rtp_mapping.encoding in supported_codecs + end + + @doc false + @spec is_supported_rtp_hdr_extension(t(), ExSDP.Attribute.Extmap.t()) :: boolean() + def is_supported_rtp_hdr_extension(config, rtp_hdr_extension) do + rtp_hdr_extension.uri in config.rtp_hdr_extensions + end + + @doc false + @spec is_supported_rtcp_feedback(t(), ExSDP.Attribute.RTCPFeedback.t()) :: boolean() + def is_supported_rtcp_feedback(_config, _rtcp_feedback), do: false + + defp add_mandatory_rtp_hdr_extensions(options) do + Keyword.update(options, :rtp_hdr_extensions, @mandatory_rtp_hdr_exts, fn exts -> + exts ++ @mandatory_rtp_hdr_exts + end) + end + + defp resolve_rtp_hdr_extensions(options) do + rtp_hdr_extensions = + Enum.map(options[:rtp_hdr_extensions], fn ext -> Map.fetch!(@rtp_hdr_extensions, ext) end) + + Keyword.put(options, :rtp_hdr_extensions, rtp_hdr_extensions) + end - # ATM, ExICE does not support relay via TURN - stun_servers = - config.ice_servers + defp resolve_codecs(options) do + end + + defp reject_turn_servers(options) do + Keyword.update(options, :ice_servers, [], fn ice_servers -> + ice_servers |> Enum.flat_map(&List.wrap(&1.urls)) |> Enum.filter(&String.starts_with?(&1, "stun:")) - - %__MODULE__{config | ice_servers: stun_servers} + end) end end diff --git a/lib/ex_webrtc/rtp_transceiver.ex b/lib/ex_webrtc/rtp_transceiver.ex index e18d5d02..4ad2ac46 100644 --- a/lib/ex_webrtc/rtp_transceiver.ex +++ b/lib/ex_webrtc/rtp_transceiver.ex @@ -3,7 +3,7 @@ defmodule ExWebRTC.RTPTransceiver do RTPTransceiver """ - alias ExWebRTC.{RTPCodecParameters, RTPReceiver} + alias ExWebRTC.{PeerConnection.Configuration, RTPCodecParameters, RTPReceiver} @type direction() :: :sendonly | :recvonly | :sendrecv | :inactive | :stopped @type kind() :: :audio | :video @@ -12,8 +12,8 @@ defmodule ExWebRTC.RTPTransceiver do mid: String.t(), direction: direction(), kind: kind(), - hdr_exts: [], - codecs: [], + hdr_exts: [ExSDP.Attribute.Extmap.t()], + codecs: [RTPCodecParameters.t()], rtp_receiver: nil } @@ -32,14 +32,14 @@ defmodule ExWebRTC.RTPTransceiver do # if it doesn't exist, creats a new one # returns list of updated transceivers @doc false - def update_or_create(transceivers, mid, mline) do + def update_or_create(transceivers, mid, mline, config) do case find_by_mid(transceivers, mid) do {idx, %__MODULE__{} = tr} -> - List.replace_at(transceivers, idx, update(tr, mline)) + List.replace_at(transceivers, idx, update(tr, mline, config)) nil -> - codecs = get_codecs(mline) - hdr_exts = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.Extmap) + codecs = get_codecs(mline, config) + hdr_exts = get_rtp_hdr_extensions(mline, config) ssrc = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.SSRC) tr = %__MODULE__{ @@ -55,26 +55,37 @@ defmodule ExWebRTC.RTPTransceiver do end end - defp update(transceiver, mline) do - codecs = get_codecs(mline) - hdr_exts = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.Extmap) + defp update(transceiver, mline, config) do + codecs = get_codecs(mline, config) + hdr_exts = get_rtp_hdr_extensions(mline, config) ssrc = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.SSRC) rtp_receiver = %RTPReceiver{ssrc: ssrc} %__MODULE__{transceiver | codecs: codecs, hdr_exts: hdr_exts, rtp_receiver: rtp_receiver} end - defp get_codecs(mline) do + defp get_codecs(mline, config) do + find_corresponding_rtcp_fbs = fn all_rtcp_fbs, rtp_mapping, config -> + all_rtcp_fbs + |> Stream.filter(&(&1.pt == rtp_mapping.payload_type)) + |> Enum.filter(&Configuration.is_supported_rtcp_feedback(config, &1)) + end + rtp_mappings = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.RTPMapping) fmtps = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.FMTP) all_rtcp_fbs = ExSDP.Media.get_attributes(mline, ExSDP.Attribute.RTCPFeedback) - for rtp_mapping <- rtp_mappings do - fmtp = Enum.find(fmtps, fn fmtp -> fmtp.pt == rtp_mapping.payload_type end) - - rtcp_fbs = - Enum.filter(all_rtcp_fbs, fn rtcp_fb -> rtcp_fb.pt == rtp_mapping.payload_type end) - + rtp_mappings + |> Stream.filter(fn rtp_mapping -> Configuration.is_supported_codec(config, rtp_mapping) end) + |> Enum.map(fn rtp_mapping -> + fmtp = Enum.find(fmtps, &(&1.pt == rtp_mapping.payload_type)) + rtcp_fbs = find_corresponding_rtcp_fbs.(all_rtcp_fbs, rtp_mapping, config) RTPCodecParameters.new(mline.type, rtp_mapping, fmtp, rtcp_fbs) - end + end) + end + + defp get_rtp_hdr_extensions(mline, config) do + mline + |> ExSDP.Media.get_attributes(ExSDP.Attribute.Extmap) + |> Enum.filter(&Configuration.is_supported_rtp_hdr_extension(config, &1)) end end