Merge pull request #11 from membraneframework/support-various-video-c…

…odecs-in-source Support various video codecs in source
membraneframework · Nov 22, 2024 · 71f23f6 · 71f23f6
2 parents 50e1242 + 7f761d3
commit 71f23f6
Show file tree

Hide file tree

Showing 13 changed files with 365 additions and 106 deletions.
diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ The package can be installed by adding `membrane_webrtc_plugin` to your list of
 ```elixir
 def deps do
   [
-    {:membrane_webrtc_plugin, "~> 0.22.1"}
+    {:membrane_webrtc_plugin, "~> 0.23.0"}
   ]
 end
 ```

diff --git a/lib/membrane_webrtc/ex_webrtc/sink.ex b/lib/membrane_webrtc/ex_webrtc/sink.ex
@@ -210,20 +210,16 @@ defmodule Membrane.WebRTC.ExWebRTCSink do
 
     %{negotiating_tracks: negotiating_tracks, negotiated_tracks: negotiated_tracks} = state
 
-    video_codecs = get_negotiated_video_codecs(sdp)
+    video_codecs = ExWebRTCUtils.get_video_codecs_from_sdp(sdp)
 
     to_notify =
-      negotiating_tracks
-      |> Enum.filter(& &1.notify)
-      |> Enum.map(&Map.take(&1, [:id, :kind]))
-      |> Enum.map(fn
-        %{kind: :audio} = track -> Map.put(track, :codec, :opus)
-        %{kind: :video} = track -> Map.put(track, :codec, video_codecs)
-      end)
+      negotiating_tracks |> Enum.filter(& &1.notify) |> Enum.map(&Map.take(&1, [:id, :kind]))
 
-    actions =
+    new_tracks_notification =
       if to_notify == [], do: [], else: [notify_parent: {:new_tracks, to_notify}]
 
+    actions = [notify_parent: {:negotiated_video_codecs, video_codecs}] ++ new_tracks_notification
+
     negotiated_tracks = negotiated_tracks ++ negotiating_tracks
 
     state =
@@ -296,20 +292,4 @@ defmodule Membrane.WebRTC.ExWebRTCSink do
     seq_num = rem(params.seq_num + 1, @max_rtp_seq_num + 1)
     put_in(state.input_tracks[pad], {id, %{params | seq_num: seq_num}})
   end
-
-  defp get_negotiated_video_codecs(sdp_answer) do
-    ex_sdp = ExSDP.parse!(sdp_answer.sdp)
-
-    ex_sdp.media
-    |> Enum.flat_map(fn
-      %{type: :video, attributes: attributes} -> attributes
-      _media -> []
-    end)
-    |> Enum.flat_map(fn
-      %ExSDP.Attribute.RTPMapping{encoding: "H264"} -> [:h264]
-      %ExSDP.Attribute.RTPMapping{encoding: "VP8"} -> [:vp8]
-      _attribute -> []
-    end)
-    |> Enum.uniq()
-  end
 end
diff --git a/lib/membrane_webrtc/ex_webrtc/source.ex b/lib/membrane_webrtc/ex_webrtc/source.ex
@@ -9,7 +9,8 @@ defmodule Membrane.WebRTC.ExWebRTCSource do
   alias Membrane.WebRTC.{ExWebRTCUtils, SignalingChannel, SimpleWebSocketServer, WhipServer}
 
   def_options signaling: [],
-              video_codec: [],
+              allowed_video_codecs: [],
+              preferred_video_codec: [],
               ice_servers: [],
               keyframe_interval: [],
               sdp_candidates_timeout: []
@@ -39,6 +40,8 @@ defmodule Membrane.WebRTC.ExWebRTCSource do
             status: :init | :connecting | :connected | :closed,
             audio_params: [ExWebRTC.RTPCodecParameters.t()],
             video_params: [ExWebRTC.RTPCodecParameters.t()],
+            allowed_video_codecs: [:h264 | :vp8],
+            preferred_video_codec: :h264 | :vp8,
             ice_servers: [ExWebRTC.PeerConnection.Configuration.ice_server()],
             keyframe_interval: Membrane.Time.t() | nil,
             sdp_candidates_timeout: Membrane.Time.t() | nil
@@ -47,13 +50,16 @@ defmodule Membrane.WebRTC.ExWebRTCSource do
     @enforce_keys [
       :signaling,
       :audio_params,
-      :video_params,
+      :allowed_video_codecs,
+      :preferred_video_codec,
       :ice_servers,
       :keyframe_interval,
       :sdp_candidates_timeout
     ]
+
     defstruct @enforce_keys ++
                 [
+                  video_params: nil,
                   pc: nil,
                   output_tracks: %{},
                   awaiting_outputs: [],
@@ -69,7 +75,8 @@ defmodule Membrane.WebRTC.ExWebRTCSource do
      %State{
        signaling: opts.signaling,
        audio_params: ExWebRTCUtils.codec_params(:opus),
-       video_params: ExWebRTCUtils.codec_params(opts.video_codec),
+       allowed_video_codecs: opts.allowed_video_codecs |> Enum.uniq(),
+       preferred_video_codec: opts.preferred_video_codec,
        ice_servers: opts.ice_servers,
        keyframe_interval: opts.keyframe_interval,
        sdp_candidates_timeout: opts.sdp_candidates_timeout
@@ -95,18 +102,10 @@ defmodule Membrane.WebRTC.ExWebRTCSource do
 
   @impl true
   def handle_playing(_ctx, state) do
-    {:ok, pc} =
-      PeerConnection.start(
-        ice_servers: state.ice_servers,
-        video_codecs: state.video_params,
-        audio_codecs: state.audio_params
-      )
-
-    Process.monitor(pc)
     Process.monitor(state.signaling.pid)
     SignalingChannel.register_element(state.signaling)
 
-    {[], %{state | pc: pc, status: :connecting}}
+    {[], %{state | status: :connecting}}
   end
 
   @impl true
@@ -216,6 +215,8 @@ defmodule Membrane.WebRTC.ExWebRTCSource do
         state
       ) do
     Membrane.Logger.debug("Received SDP offer")
+
+    {codecs_notification, state} = ensure_peer_connection_started(sdp, state)
     :ok = PeerConnection.set_remote_description(state.pc, sdp)
 
     {new_tracks, awaiting_outputs} =
@@ -264,7 +265,7 @@ defmodule Membrane.WebRTC.ExWebRTCSource do
           []
       end)
 
-    {tracks_notification ++ stream_formats, state}
+    {codecs_notification ++ tracks_notification ++ stream_formats, state}
   end
 
   @impl true
@@ -303,6 +304,35 @@ defmodule Membrane.WebRTC.ExWebRTCSource do
     {[], state}
   end
 
+  defp ensure_peer_connection_started(sdp, %{pc: nil} = state) do
+    video_codecs_in_sdp = ExWebRTCUtils.get_video_codecs_from_sdp(sdp)
+
+    negotiated_video_codecs =
+      state.allowed_video_codecs
+      |> Enum.filter(&(&1 in video_codecs_in_sdp))
+      |> case do
+        [] -> []
+        [codec] -> [codec]
+        _both -> [state.preferred_video_codec]
+      end
+
+    video_params = ExWebRTCUtils.codec_params(negotiated_video_codecs)
+
+    {:ok, pc} =
+      PeerConnection.start(
+        ice_servers: state.ice_servers,
+        video_codecs: video_params,
+        audio_codecs: state.audio_params
+      )
+
+    Process.monitor(pc)
+
+    notify_parent = [notify_parent: {:negotiated_video_codecs, negotiated_video_codecs}]
+    {notify_parent, %{state | pc: pc, video_params: video_params}}
+  end
+
+  defp ensure_peer_connection_started(_sdp, state), do: {[], state}
+
   defp maybe_answer(state) do
     if Enum.all?(state.output_tracks, fn {_id, %{status: status}} -> status == :connected end) do
       %{pc: pc} = state

diff --git a/lib/membrane_webrtc/ex_webrtc/utils.ex b/lib/membrane_webrtc/ex_webrtc/utils.ex
@@ -61,4 +61,21 @@ defmodule Membrane.WebRTC.ExWebRTCUtils do
         90_000
     end
   end
+
+  @spec get_video_codecs_from_sdp(ExWebRTC.SessionDescription.t()) :: [:h264 | :vp8]
+  def get_video_codecs_from_sdp(%ExWebRTC.SessionDescription{sdp: sdp}) do
+    ex_sdp = ExSDP.parse!(sdp)
+
+    ex_sdp.media
+    |> Enum.flat_map(fn
+      %{type: :video, attributes: attributes} -> attributes
+      _media -> []
+    end)
+    |> Enum.flat_map(fn
+      %ExSDP.Attribute.RTPMapping{encoding: "H264"} -> [:h264]
+      %ExSDP.Attribute.RTPMapping{encoding: "VP8"} -> [:vp8]
+      _attribute -> []
+    end)
+    |> Enum.uniq()
+  end
 end
diff --git a/...membrane_webrtc/sink/forwarding_filter.ex → lib/membrane_webrtc/forwarding_filter.ex b/...membrane_webrtc/sink/forwarding_filter.ex → lib/membrane_webrtc/forwarding_filter.ex
@@ -1,4 +1,4 @@
-defmodule Membrane.WebRTC.Sink.ForwardingFilter do
+defmodule Membrane.WebRTC.ForwardingFilter do
   @moduledoc false
   use Membrane.Filter
 

diff --git a/lib/membrane_webrtc/sink.ex b/lib/membrane_webrtc/sink.ex
@@ -17,12 +17,10 @@ defmodule Membrane.WebRTC.Sink do
   """
   use Membrane.Bin
 
-  alias __MODULE__.ForwardingFilter
-
   alias Membrane.H264
   alias Membrane.RemoteStream
   alias Membrane.VP8
-  alias Membrane.WebRTC.{ExWebRTCSink, SignalingChannel, SimpleWebSocketServer}
+  alias Membrane.WebRTC.{ExWebRTCSink, ForwardingFilter, SignalingChannel, SimpleWebSocketServer}
 
   @typedoc """
   Notification that should be sent to the bin to negotiate new tracks.
@@ -184,13 +182,9 @@ defmodule Membrane.WebRTC.Sink do
   end
 
   @impl true
-  def handle_child_notification({:new_tracks, tracks}, :webrtc, _ctx, state) do
-    {[notify_parent: {:new_tracks, tracks}], state}
-  end
-
-  @impl true
-  def handle_child_notification({:negotiated_video_codecs, codecs}, :webrtc, _ctx, state) do
-    {[notify_parent: {:negotiated_video_codecs, codecs}], state}
+  def handle_child_notification({type, _content} = notification, :webrtc, _ctx, state)
+      when type in [:new_tracks, :negotiated_video_codecs] do
+    {[notify_parent: notification], state}
   end
 
   @impl true