adding rtcp processing example

elixir-webrtc · Feb 21, 2024 · b895d50 · b895d50
1 parent f96e563
commit b895d50
Show file tree

Hide file tree

Showing 10 changed files with 392 additions and 0 deletions.
diff --git a/examples/rtcp-processing/.formatter.exs b/examples/rtcp-processing/.formatter.exs
@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+]
diff --git a/examples/rtcp-processing/.gitignore b/examples/rtcp-processing/.gitignore
@@ -0,0 +1,26 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+echo-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/
diff --git a/examples/rtcp-processing/README.md b/examples/rtcp-processing/README.md
@@ -0,0 +1,13 @@
+# RTCP Processing
+
+Receive video and audio from a browser and display the RTCP packets in the browser
+
+rtcp-processing demonstrates the Public API for processing RTCP packets in ExWebRTC.
+
+RTCP is used for statistics and control information for media in WebRTC. Using these messages you can get information about the quality of the media, round trip time and packet loss. You can also craft messages to influence the media quality.
+
+While in `examples/rtcp-processing` directory
+
+1. Run `mix deps.get`
+2. Run `mix run --no-halt`
+3. Visit `http://127.0.0.1:8829/index.html` in your browser and press the `play` button.
diff --git a/examples/rtcp-processing/assets/index.html b/examples/rtcp-processing/assets/index.html
@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <meta http-equiv="X-UA-Compatible" content="ie=edge">
+  <title>Elixir WebRTC RTCP Example</title>
+</head>
+
+<body>
+  <main>
+    <h1>Elixir WebRTC RTCP Example</h1>
+  </main>
+  <video id="videoPlayer" autoplay controls></video>
+  <h2>RTCP Packets from the server:</h2>
+  <div id="rtcpPackets"></div>
+  <script src="script.js"></script>
+</body>
+
+</html>
diff --git a/examples/rtcp-processing/assets/script.js b/examples/rtcp-processing/assets/script.js
@@ -0,0 +1,68 @@
+const pcConfig = { 'iceServers': [{ 'urls': 'stun:stun.l.google.com:19302' },] };
+const mediaConstraints = { video: true, audio: true }
+
+const proto = window.location.protocol === "https:" ? "wss:" : "ws:"
+const ws = new WebSocket(`${proto}//${window.location.host}/ws`);
+ws.onopen = _ => start_connection(ws);
+ws.onclose = event => console.log("WebSocket connection was terminated:", event);
+const start_connection = async (ws) => {
+  const videoPlayer = document.getElementById("videoPlayer");
+  const rtcpPacketsEl = document.getElementById("rtcpPackets");
+  videoPlayer.srcObject = new MediaStream();
+
+  const pc = new RTCPeerConnection(pcConfig);
+  pc.ontrack = event => videoPlayer.srcObject.addTrack(event.track);
+  pc.onicecandidate = event => {
+    if (event.candidate === null) return;
+
+    console.log("Sent ICE candidate:", event.candidate);
+    ws.send(JSON.stringify({ type: "ice", data: event.candidate }));
+  };
+
+  const localStream = await navigator.mediaDevices.getUserMedia(mediaConstraints);
+  for (const track of localStream.getTracks()) {
+    pc.addTrack(track, localStream);
+  }
+  pc.addEventListener(
+    "connectionstatechange",
+    (_event) => {
+      if (pc.connectionState == "connected") {
+        setInterval(function() {
+          ws.send(JSON.stringify({ type: "getRTCP" }))
+        }, 1000)
+      }
+    },
+    false,
+  );
+
+  ws.onmessage = async event => {
+    console.debug('[message] Data received from server: ', event.data);
+
+    try {
+      const { type, data } = JSON.parse(event.data);
+
+      switch (type) {
+        case "answer":
+          console.log("Received SDP answer:", data);
+          await pc.setRemoteDescription(data)
+          break;
+        case "ice":
+          console.log("Recieved ICE candidate:", data);
+          await pc.addIceCandidate(data);
+        case "rtcpPackets":
+          console.log("Recieved rtcp packets:", data);
+          var p = document.createElement('p');
+          p.innerHTML = JSON.stringify(data)
+          rtcpPacketsEl.appendChild(p)
+
+      }
+    } catch (error) {
+      console.error(error);
+    }
+  };
+
+  const offer = await pc.createOffer();
+  await pc.setLocalDescription(offer);
+  console.log("Sent SDP offer:", offer)
+  ws.send(JSON.stringify({ type: "offer", data: offer }));
+};
diff --git a/examples/rtcp-processing/lib/rtcp_processing/application.ex b/examples/rtcp-processing/lib/rtcp_processing/application.ex
@@ -0,0 +1,19 @@
+defmodule RtcpProcessing.Application do
+  use Application
+
+  require Logger
+
+  @ip {127, 0, 0, 1}
+  @port 8829
+
+  @impl true
+  def start(_type, _args) do
+    Logger.configure(level: :info)
+
+    children = [
+      {Bandit, plug: RtcpProcessing.Router, ip: @ip, port: @port}
+    ]
+
+    Supervisor.start_link(children, strategy: :one_for_one)
+  end
+end
diff --git a/examples/rtcp-processing/lib/rtcp_processing/peer_handler.ex b/examples/rtcp-processing/lib/rtcp_processing/peer_handler.ex
@@ -0,0 +1,159 @@
+defmodule RtcpProcessing.PeerHandler do
+  require Logger
+
+  alias ExWebRTC.{
+    ICECandidate,
+    MediaStreamTrack,
+    PeerConnection,
+    RTPCodecParameters,
+    SessionDescription
+  }
+
+  @behaviour WebSock
+
+  @ice_servers [
+    %{urls: "stun:stun.l.google.com:19302"}
+  ]
+
+  @video_codecs [
+    %RTPCodecParameters{
+      payload_type: 96,
+      mime_type: "video/VP8",
+      clock_rate: 90_000
+    }
+  ]
+
+  @audio_codecs [
+    %RTPCodecParameters{
+      payload_type: 111,
+      mime_type: "audio/opus",
+      clock_rate: 48_000,
+      channels: 2
+    }
+  ]
+
+  @impl true
+  def init(_) do
+    {:ok, pc} =
+      PeerConnection.start_link(
+        ice_servers: @ice_servers,
+        video_codecs: @video_codecs,
+        audio_codecs: @audio_codecs
+      )
+
+    video_track = MediaStreamTrack.new(:video)
+    audio_track = MediaStreamTrack.new(:audio)
+
+    {:ok, _sender} = PeerConnection.add_track(pc, video_track)
+    {:ok, _sender} = PeerConnection.add_track(pc, audio_track)
+
+    state = %{
+      peer_connection: pc,
+      out_video_track_id: video_track.id,
+      out_audio_track_id: audio_track.id,
+      in_video_track_id: nil,
+      in_audio_track_id: nil,
+      rtcp_packets: []
+    }
+
+    {:ok, state}
+  end
+
+  @impl true
+  def handle_in({msg, [opcode: :text]}, state) do
+    msg
+    |> Jason.decode!()
+    |> handle_ws_msg(state)
+  end
+
+  @impl true
+  def handle_info({:ex_webrtc, _from, msg}, state) do
+    handle_webrtc_msg(msg, state)
+  end
+
+  @impl true
+  def terminate(reason, _state) do
+    Logger.warning("WebSocket connection was terminated, reason: #{inspect(reason)}")
+  end
+
+  defp handle_ws_msg(%{"type" => "offer", "data" => data}, state) do
+    Logger.info("Received SDP offer: #{inspect(data)}")
+
+    offer = SessionDescription.from_json(data)
+    :ok = PeerConnection.set_remote_description(state.peer_connection, offer)
+
+    {:ok, answer} = PeerConnection.create_answer(state.peer_connection)
+    :ok = PeerConnection.set_local_description(state.peer_connection, answer)
+
+    answer_json = SessionDescription.to_json(answer)
+
+    msg =
+      %{"type" => "answer", "data" => answer_json}
+      |> Jason.encode!()
+
+    Logger.info("Sent SDP answer: #{inspect(answer_json)}")
+
+    {:push, {:text, msg}, state}
+  end
+
+  defp handle_ws_msg(%{"type" => "ice", "data" => data}, state) do
+    Logger.info("Received ICE candidate: #{inspect(data)}")
+
+    candidate = ICECandidate.from_json(data)
+    :ok = PeerConnection.add_ice_candidate(state.peer_connection, candidate)
+    {:ok, state}
+  end
+
+  defp handle_ws_msg(%{"type" => "getRTCP"}, state) do
+    Logger.info("Received request for RTCP packets")
+
+    packets =
+      Enum.map(state.rtcp_packets, fn packet ->
+        "packet: #{inspect(packet)}"
+      end)
+
+    msg = Jason.encode!(%{type: "rtcpPackets", data: packets})
+
+    {:push, {:text, msg}, %{state | rtcp_packets: []}}
+  end
+
+  defp handle_webrtc_msg({:ice_candidate, candidate}, state) do
+    candidate_json = ICECandidate.to_json(candidate)
+
+    msg =
+      %{"type" => "ice", "data" => candidate_json}
+      |> Jason.encode!()
+
+    Logger.info("Sent ICE candidate: #{inspect(candidate_json)}")
+
+    {:push, {:text, msg}, state}
+  end
+
+  defp handle_webrtc_msg({:track, track}, state) do
+    %MediaStreamTrack{kind: kind, id: id} = track
+
+    state =
+      case kind do
+        :video -> %{state | in_video_track_id: id}
+        :audio -> %{state | in_audio_track_id: id}
+      end
+
+    {:ok, state}
+  end
+
+  defp handle_webrtc_msg({:rtp, id, packet}, %{in_audio_track_id: id} = state) do
+    PeerConnection.send_rtp(state.peer_connection, state.out_audio_track_id, packet)
+    {:ok, state}
+  end
+
+  defp handle_webrtc_msg({:rtp, id, packet}, %{in_video_track_id: id} = state) do
+    PeerConnection.send_rtp(state.peer_connection, state.out_video_track_id, packet)
+    {:ok, state}
+  end
+
+  defp handle_webrtc_msg({:rtcp, packets}, state) do
+    {:ok, %{state | rtcp_packets: state.rtcp_packets ++ packets}}
+  end
+
+  defp handle_webrtc_msg(_msg, state), do: {:ok, state}
+end
diff --git a/examples/rtcp-processing/lib/rtcp_processing/router.ex b/examples/rtcp-processing/lib/rtcp_processing/router.ex
@@ -0,0 +1,15 @@
+defmodule RtcpProcessing.Router do
+  use Plug.Router
+
+  plug(Plug.Static, at: "/", from: "assets")
+  plug(:match)
+  plug(:dispatch)
+
+  get "/ws" do
+    WebSockAdapter.upgrade(conn, RtcpProcessing.PeerHandler, %{}, [])
+  end
+
+  match _ do
+    send_resp(conn, 404, "not found")
+  end
+end
diff --git a/examples/rtcp-processing/mix.exs b/examples/rtcp-processing/mix.exs
@@ -0,0 +1,30 @@
+defmodule RtcpProcessing.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :rtcp_processing,
+      version: "0.1.0",
+      elixir: "~> 1.15",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [
+      extra_applications: [:logger],
+      mod: {RtcpProcessing.Application, []}
+    ]
+  end
+
+  defp deps do
+    [
+      {:plug, "~> 1.15.0"},
+      {:bandit, "~> 1.2.0"},
+      {:websock_adapter, "~> 0.5.0"},
+      {:jason, "~> 1.4.0"},
+      {:ex_webrtc, path: "../../."}
+    ]
+  end
+end