Skip to content

Commit

Permalink
Add set_sender_codec (#190)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mickel8
mickel8 authored Feb 7, 2025
1 parent 0c930f2 commit 053d520
Show file tree
Hide file tree
Showing 11 changed files with 821 additions and 287 deletions.
87 changes: 86 additions & 1 deletion lib/ex_webrtc/peer_connection.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ defmodule ExWebRTC.PeerConnection do

require Logger

alias ExWebRTC.RTPCodecParameters
alias __MODULE__.{Configuration, Demuxer, TWCCRecorder}

alias ExWebRTC.{
Expand Down Expand Up @@ -152,9 +153,67 @@ defmodule ExWebRTC.PeerConnection do
GenServer.call(peer_connection, :get_configuration)
end

@doc """
Sets the codec that will be used for sending RTP packets.
`send_rtp/4` overrides some of the RTP packet fields.
In particular, when multiple codecs are negotiated, `send_rtp/4` will use
payload type of the most preffered by the remote side codec (i.e. the first
one from the list of codecs in the remote description).
Use this function if you want to select, which codec (hence payload type)
should be used for sending.
Once the first RTP packet is sent (via `send_rtp/4`), `set_sender_codec/3`
can only be called with a codec with the same clock rate.
Although very unlikely, keep in mind that after renegotiation,
the selected codec may no longer be supported by the remote side and you might
need to call this function again, passing a new codec.
To check available codecs you can use `get_transceivers/1`:
```
{:ok, pc} = PeerConnection.start_link()
{:ok, rtp_sender} = PeerConnection.add_track(MediaStreamTrack.new(:video))
# exchange SDP with the remote side
# {:ok, offer} = PeerConnection.create_offer(pc)
# ...
tr =
pc
|> PeerConnection.get_transceivers()
|> Enum.find(fn tr -> tr.sender.id == rtp_sender.id end)
dbg(tr.codecs) # list of supported codecs both for sending and receiving
# e.g. always prefer h264 over vp8
h264 = Enum.find(tr.codecs, fn codec -> codec.mime_type == "video/H264" end)
vp8 = Enum.find(tr.codecs, fn codec -> codec.mime_type == "video/VP8" end)
:ok = PeerConnection.set_sender_codec(pc, rtp_sender.id, h264 || vp8)
```
This function can only be called once the first negotiation passes.
"""
@spec set_sender_codec(peer_connection(), RTPSender.id(), RTPCodecParameters.t()) ::
:ok | {:error, term()}
def set_sender_codec(peer_connection, sender_id, codec) do
GenServer.call(peer_connection, {:set_sender_codec, sender_id, codec})
end

@doc """
Sends an RTP packet to the remote peer using the track specified by the `track_id`.
The following fields of the RTP packet will be overwritten by this function:
* payload type
* ssrc
* rtp header extensions
If you negotiated multiple codecs (hence payload types) and you want to choose,
which one should be used, see `set_sender_codec/3`.
Options:
* `rtx?` - send the packet as if it was retransmitted (use SSRC and payload type specific to RTX)
"""
Expand Down Expand Up @@ -576,6 +635,31 @@ defmodule ExWebRTC.PeerConnection do
{:reply, state.config, state}
end

@impl true
def handle_call({:set_sender_codec, sender_id, codec}, _from, state) do
state.transceivers
|> Enum.with_index()
|> Enum.find(fn {tr, _idx} -> tr.sender.id == sender_id end)
|> case do
{tr, idx} when tr.direction in [:sendrecv, :sendonly] ->
case RTPTransceiver.set_sender_codec(tr, codec) do
{:ok, tr} ->
transceivers = List.replace_at(state.transceivers, idx, tr)
state = %{state | transceivers: transceivers}
{:reply, :ok, state}

{:error, _reason} = error ->
{:reply, error, state}
end

{_tr, _idx} ->
{:reply, {:error, :invalid_transceiver_direction}, state}

nil ->
{:reply, {:error, :invalid_sender_id}, state}
end
end

@impl true
def handle_call(:get_connection_state, _from, state) do
{:reply, state.conn_state, state}
Expand Down Expand Up @@ -1135,7 +1219,8 @@ defmodule ExWebRTC.PeerConnection do
end

{packet, tr} = RTPTransceiver.send_packet(tr, packet, rtx?)
:ok = DTLSTransport.send_rtp(state.dtls_transport, packet)

if packet != <<>>, do: :ok = DTLSTransport.send_rtp(state.dtls_transport, packet)

transceivers = List.replace_at(state.transceivers, idx, tr)
state = %{state | transceivers: transceivers}
Expand Down
176 changes: 140 additions & 36 deletions lib/ex_webrtc/peer_connection/configuration.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,22 @@ defmodule ExWebRTC.PeerConnection.Configuration do
clock_rate: 90_000,
sdp_fmtp_line: %FMTP{
pt: 98,
level_asymmetry_allowed: 1,
level_asymmetry_allowed: true,
packetization_mode: 0,
profile_level_id: 0x42E01F
}
},
%RTPCodecParameters{
payload_type: 99,
mime_type: "video/H264",
clock_rate: 90_000,
sdp_fmtp_line: %FMTP{
pt: 99,
level_asymmetry_allowed: true,
packetization_mode: 1,
profile_level_id: 0x42E01F
}
},
%RTPCodecParameters{
payload_type: 45,
mime_type: "video/AV1",
Expand Down Expand Up @@ -252,10 +263,27 @@ defmodule ExWebRTC.PeerConnection.Configuration do
|> Keyword.put(:audio_extensions, Enum.map(audio_extensions, fn {_, ext} -> ext end))
|> Keyword.put(:video_extensions, Enum.map(video_extensions, fn {_, ext} -> ext end))
|> then(&struct(__MODULE__, &1))
|> ensure_unique_payload_types()
|> populate_feedbacks(feedbacks)
|> add_features()
end

defp ensure_unique_payload_types(config) do
audio_pt = Enum.map(config.audio_codecs, fn codec -> codec.payload_type end)

if length(audio_pt) != length(Enum.uniq(audio_pt)) do
raise "Payload types in audio codecs are not unique."
end

video_pt = Enum.map(config.video_codecs, fn codec -> codec.payload_type end)

if length(video_pt) != length(Enum.uniq(video_pt)) do
raise "Payload types in video codecs are not unique."
end

config
end

defp add_features(config) do
%__MODULE__{features: features} = config

Expand Down Expand Up @@ -405,6 +433,9 @@ defmodule ExWebRTC.PeerConnection.Configuration do
config
|> update_extensions(sdp)
|> update_codecs(sdp)
# if update went wrong (there are duplicates in payload types),
# we should never continue as this may lead to hard to debug errors
|> ensure_unique_payload_types()
end

defp update_extensions(config, sdp) do
Expand All @@ -425,25 +456,41 @@ defmodule ExWebRTC.PeerConnection.Configuration do
defp do_update_extensions(extensions, sdp_extensions, free_ids) do
# we replace extension ids in config to ids from the SDP
# in case we have an extension in config but not in SDP, we replace
# its id to some free (not present in SDP) id, so it doesn't conflict
# its id only when it's occupied to some free (not present in SDP) id, so it doesn't conflict
Enum.map_reduce(extensions, free_ids, fn ext, free_ids ->
sdp_extensions
|> Enum.find(&(&1.uri == ext.uri))
|> case do
nil ->
case find_in_sdp_rtp_extensions(sdp_extensions, ext) do
{nil, false} ->
{ext, free_ids}

{nil, true} ->
[id | rest] = free_ids
{%Extmap{ext | id: id}, rest}

other ->
{other, _id_used} ->
{%Extmap{ext | id: other.id}, free_ids}
end
end)
end

# Searches for rtp extension in sdp rtp extensions.
# If ext is not found, id_used determines whether ext's id
# is already present in sdp_extensions.
# Otherwise, id_used can have any value.
defp find_in_sdp_rtp_extensions(sdp_extensions, ext, id_used \\ false)
defp find_in_sdp_rtp_extensions([], _ext, id_used), do: {nil, id_used}

defp find_in_sdp_rtp_extensions([sdp_ext | sdp_extensions], ext, id_used) do
if sdp_ext.uri == ext.uri do
{sdp_ext, id_used}
else
find_in_sdp_rtp_extensions(sdp_extensions, ext, id_used || sdp_ext.id == ext.id)
end
end

defp update_codecs(config, sdp) do
%__MODULE__{audio_codecs: audio_codecs, video_codecs: video_codecs} = config
sdp_codecs = SDPUtils.get_rtp_codec_parameters(sdp)
free_pts = get_free_payload_types(sdp_codecs)
free_pts = get_free_payload_types(audio_codecs ++ video_codecs ++ sdp_codecs)

{audio_codecs, free_pts} = do_update_codecs(audio_codecs, sdp_codecs, free_pts)
{video_codecs, _free_pts} = do_update_codecs(video_codecs, sdp_codecs, free_pts)
Expand All @@ -452,29 +499,27 @@ defmodule ExWebRTC.PeerConnection.Configuration do
end

defp do_update_codecs(codecs, sdp_codecs, free_pts) do
# we replace codec payload types in config to payload types from SDP
# both normal codecs and rtx (we also update apt FMTP attribute in rtxs)
# other codecs that are present in config but not in SDP
# are also updated with values from a pool of free payload types (not present in SDP)
# to make sure they don't conflict
{sdp_rtxs, sdp_codecs} = Enum.split_with(sdp_codecs, &rtx?/1)
# We replace codec payload types in config to payload types from SDP
# both for normal codecs and rtx (we also update apt FMTP attribute in rtxs).
# Other codecs that are present in config but not in SDP, and their
# payload type is already present in SDP, are also updated with values
# from a pool of free payload types (not present in SDP) to make sure they don't conflict
{rtxs, codecs} = Enum.split_with(codecs, &rtx?/1)

{codecs, {free_pts, mapping}} =
Enum.map_reduce(codecs, {free_pts, %{}}, fn codec, {free_pts, mapping} ->
sdp_codecs
|> Enum.find(
&(String.downcase(&1.mime_type) == String.downcase(codec.mime_type) and
&1.clock_rate == codec.clock_rate and
&1.channels == codec.channels)
)
|> case do
nil ->
case find_in_sdp_codecs(sdp_codecs, codec) do
# there is no such codec and its payload type is not used
{nil, false} ->
{codec, {free_pts, Map.put(mapping, codec.payload_type, codec.payload_type)}}

# there is no such codec, but its payload type is used
{nil, true} ->
[pt | rest] = free_pts
new_codec = do_update_codec(codec, pt)
{new_codec, {rest, Map.put(mapping, codec.payload_type, pt)}}

other ->
{other, _pt_used} ->
new_codec = do_update_codec(codec, other.payload_type)
{new_codec, {free_pts, Map.put(mapping, codec.payload_type, other.payload_type)}}
end
Expand All @@ -486,15 +531,18 @@ defmodule ExWebRTC.PeerConnection.Configuration do
%RTPCodecParameters{rtx | sdp_fmtp_line: %FMTP{fmtp | apt: Map.fetch!(mapping, apt)}}
end)
|> Enum.map_reduce(free_pts, fn rtx, free_pts ->
sdp_rtxs
|> Enum.find(&(&1.sdp_fmtp_line.apt == rtx.sdp_fmtp_line.apt))
|> case do
nil ->
case find_in_sdp_codecs(sdp_codecs, rtx) do
# there is no such codec and its payload type is not used
{nil, false} ->
{rtx, free_pts}

# there is no such codec, but its payload type is used
{nil, true} ->
[pt | rest] = free_pts
rtx = do_update_codec(rtx, pt)
{rtx, rest}

other ->
{other, _pt_used} ->
rtx = do_update_codec(rtx, other.payload_type)
{rtx, free_pts}
end
Expand All @@ -503,6 +551,38 @@ defmodule ExWebRTC.PeerConnection.Configuration do
{codecs ++ rtxs, free_pts}
end

# Searches for codec in sdp_codecs.
# If codec is not found, pt_used determines whether
# codec's payload type is already present in sdp_codecs.
# Otherwise, pt_used can have any value.
defp find_in_sdp_codecs(sdp_codecs, codec, pt_used \\ false)

defp find_in_sdp_codecs([], _codec, pt_used), do: {nil, pt_used}

defp find_in_sdp_codecs([sdp_codec | sdp_codecs], codec, pt_used) do
if String.ends_with?(codec.mime_type, "/rtx") do
if sdp_codec.sdp_fmtp_line != nil && sdp_codec.sdp_fmtp_line.apt == codec.sdp_fmtp_line.apt do
{sdp_codec, pt_used}
else
find_in_sdp_codecs(
sdp_codecs,
codec,
pt_used || sdp_codec.payload_type == codec.payload_type
)
end
else
if codec_equal_soft?(sdp_codec, codec) do
{sdp_codec, pt_used}
else
find_in_sdp_codecs(
sdp_codecs,
codec,
pt_used || sdp_codec.payload_type == codec.payload_type
)
end
end
end

defp do_update_codec(codec, new_pt) do
%RTPCodecParameters{rtcp_fbs: fbs, sdp_fmtp_line: fmtp} = codec
new_fbs = Enum.map(fbs, &%RTCPFeedback{&1 | pt: new_pt})
Expand All @@ -515,7 +595,7 @@ defmodule ExWebRTC.PeerConnection.Configuration do
def intersect_codecs(config, mline) do
# we assume that this function is called after
# the config was updated based on the remote SDP
# so the payload types should match
# so the payload types (in codec_equal?) should match
codecs =
case mline.type do
:audio -> config.audio_codecs
Expand All @@ -526,13 +606,7 @@ defmodule ExWebRTC.PeerConnection.Configuration do
|> SDPUtils.get_rtp_codec_parameters()
|> Enum.flat_map(fn sdp_codec ->
codecs
|> Enum.find(
# as of now, we ignore sdp_fmtp_line
&(String.downcase(&1.mime_type) == String.downcase(sdp_codec.mime_type) and
&1.payload_type == sdp_codec.payload_type and
&1.clock_rate == sdp_codec.clock_rate and
&1.channels == sdp_codec.channels)
)
|> Enum.find(&codec_equal?(&1, sdp_codec))
|> case do
nil ->
[]
Expand All @@ -544,6 +618,36 @@ defmodule ExWebRTC.PeerConnection.Configuration do
end)
end

# soft functions does not compare payload types
@doc false
@spec codec_equal?(RTPCodecParameters.t(), RTPCodecParameters.t()) :: boolean()
def codec_equal?(c1, c2) do
String.downcase(c1.mime_type) == String.downcase(c2.mime_type) and
c1.payload_type == c2.payload_type and
c1.clock_rate == c2.clock_rate and
c1.channels == c2.channels and fmtp_equal?(c1, c2)
end

defp codec_equal_soft?(c1, c2) do
String.downcase(c1.mime_type) == String.downcase(c2.mime_type) and
c1.clock_rate == c2.clock_rate and
c1.channels == c2.channels and fmtp_equal_soft?(c1, c2)
end

defp fmtp_equal?(%{sdp_fmtp_line: nil}, _c2), do: true
defp fmtp_equal?(_c1, %{sdp_fmtp_line: nil}), do: true
defp fmtp_equal?(c1, c2), do: c1.sdp_fmtp_line == c2.sdp_fmtp_line

defp fmtp_equal_soft?(%{sdp_fmtp_line: nil}, _c2), do: true
defp fmtp_equal_soft?(_c1, %{sdp_fmtp_line: nil}), do: true

defp fmtp_equal_soft?(c1, c2) do
fmtp1 = %{c1.sdp_fmtp_line | pt: nil}
fmtp2 = %{c2.sdp_fmtp_line | pt: nil}

fmtp1 == fmtp2
end

@doc false
@spec intersect_extensions(t(), ExSDP.Media.t()) :: [Extmap.t()]
def intersect_extensions(config, mline) do
Expand Down
Loading

0 comments on commit 053d520

Please sign in to comment.