diff --git a/lib/ex_webrtc/dtls_transport.ex b/lib/ex_webrtc/dtls_transport.ex
new file mode 100644
index 00000000..546b9392
--- /dev/null
+++ b/lib/ex_webrtc/dtls_transport.ex
@@ -0,0 +1,140 @@
+defmodule ExWebRTC.DTLSTransport do
+  @moduledoc false
+
+  require Logger
+
+  alias ExICE.ICEAgent
+
+  defstruct [
+    :ice_agent,
+    :ice_state,
+    :client,
+    :buffered_packets,
+    :cert,
+    :pkey,
+    :fingerprint,
+    :mode,
+    finished: false,
+    should_start: false
+  ]
+
+  def new(ice_agent) do
+    # temporary hack to generate certs
+    {:ok, cert_client} = ExDTLS.start_link(client_mode: true, dtls_srtp: true)
+    {:ok, cert} = ExDTLS.get_cert(cert_client)
+    {:ok, pkey} = ExDTLS.get_pkey(cert_client)
+    {:ok, fingerprint} = ExDTLS.get_cert_fingerprint(cert_client)
+    :ok = ExDTLS.stop(cert_client)
+
+    %__MODULE__{
+      ice_agent: ice_agent,
+      cert: cert,
+      pkey: pkey,
+      fingerprint: fingerprint
+    }
+  end
+
+  def start(dtls, :passive) do
+    {:ok, client} =
+      ExDTLS.start_link(
+        client_mode: false,
+        dtls_srtp: true,
+        pkey: dtls.pkey,
+        cert: dtls.cert
+      )
+
+    %__MODULE__{dtls | client: client}
+  end
+
+  def start(%{ice_state: ice_state} = dtls, :active) do
+    {:ok, client} =
+      ExDTLS.start_link(
+        client_mode: true,
+        dtls_srtp: true,
+        pkey: dtls.pkey,
+        cert: dtls.cert
+      )
+
+    dtls = %__MODULE__{dtls | client: client}
+
+    case ice_state do
+      state when state in [:active, :connected] ->
+        start_handshake(dtls)
+        dtls
+
+      _other ->
+        %__MODULE__{dtls | should_start: true}
+    end
+  end
+
+  def update_ice_state(dtls, :connected) do
+    dtls =
+      if dtls.should_start do
+        start_handshake(dtls)
+        %__MODULE__{dtls | should_start: false}
+      else
+        dtls
+      end
+
+    dtls =
+      if dtls.buffered_packets do
+        Logger.debug("Sending buffered DTLS packets")
+        ICEAgent.send_data(dtls.ice_agent, dtls.buffered_packets)
+        %__MODULE__{dtls | buffered_packets: nil}
+      else
+        dtls
+      end
+
+    %__MODULE__{dtls | ice_state: :connected}
+  end
+
+  def update_ice_state(dtls, new_state) do
+    %__MODULE__{dtls | ice_state: new_state}
+  end
+
+  def handle_info(dtls, {:retransmit, packets})
+      when dtls.ice_state in [:connected, :completed] do
+    ICEAgent.send_data(dtls.ice_agent, packets)
+    dtls
+  end
+
+  def handle_info(%{buffered_packets: packets} = dtls, {:retransmit, packets}) do
+    # we got DTLS packets from the other side but
+    # we haven't established ICE connection yet so
+    # packets to retransmit have to be the same as dtls_buffered_packets
+    dtls
+  end
+
+  def process_data(dtls, data) do
+    case ExDTLS.process(dtls.client, data) do
+      {:handshake_packets, packets} when dtls.ice_state in [:connected, :completed] ->
+        :ok = ICEAgent.send_data(dtls.ice_agent, packets)
+        dtls
+
+      {:handshake_packets, packets} ->
+        Logger.debug("""
+        Generated local DTLS packets but ICE is not in the connected or completed state yet.
+        We will send those packets once ICE is ready.
+        """)
+
+        %__MODULE__{dtls | buffered_packets: packets}
+
+      {:handshake_finished, _keying_material, packets} ->
+        Logger.debug("DTLS handshake finished")
+        ICEAgent.send_data(dtls.ice_agent, packets)
+        %__MODULE__{dtls | finished: true}
+
+      {:handshake_finished, _keying_material} ->
+        Logger.debug("DTLS handshake finished")
+        %__MODULE__{dtls | finished: true}
+
+      :handshake_want_read ->
+        dtls
+    end
+  end
+
+  defp start_handshake(dtls) do
+    {:ok, packets} = ExDTLS.do_handshake(dtls.client)
+    :ok = ICEAgent.send_data(dtls.ice_agent, packets)
+  end
+end
diff --git a/lib/ex_webrtc/peer_connection.ex b/lib/ex_webrtc/peer_connection.ex
index 165b9ba5..ec2a2528 100644
--- a/lib/ex_webrtc/peer_connection.ex
+++ b/lib/ex_webrtc/peer_connection.ex
@@ -9,6 +9,7 @@ defmodule ExWebRTC.PeerConnection do
   alias ExICE.ICEAgent
 
   alias ExWebRTC.{
+    DTLSTransport,
     IceCandidate,
     MediaStreamTrack,
     RTPTransceiver,
@@ -37,10 +38,7 @@ defmodule ExWebRTC.PeerConnection do
                 :pending_remote_desc,
                 :ice_agent,
                 :ice_state,
-                :dtls_buffered_packets,
-                dtls_client: nil,
-                dtls_finished: false,
-                initiate_dtls: false,
+                :dtls_transport,
                 transceivers: [],
                 signaling_state: :stable,
                 last_offer: nil,
@@ -115,11 +113,13 @@ defmodule ExWebRTC.PeerConnection do
       |> Enum.filter(&String.starts_with?(&1, "stun:"))
 
     {:ok, ice_agent} = ICEAgent.start_link(:controlled, stun_servers: stun_servers)
+    dtls_transport = DTLSTransport.new(ice_agent)
 
     state = %__MODULE__{
       owner: owner,
       config: config,
-      ice_agent: ice_agent
+      ice_agent: ice_agent,
+      dtls_transport: dtls_transport
     }
 
     {:ok, state}
@@ -144,13 +144,6 @@ defmodule ExWebRTC.PeerConnection do
 
     {:ok, ice_ufrag, ice_pwd} = ICEAgent.get_local_credentials(state.ice_agent)
 
-    # TODO this will fail on subsequent calls
-    # or when answer contains setup: passive
-    {:ok, dtls_client} = ExDTLS.start_link(client_mode: false, dtls_srtp: true)
-    state = %__MODULE__{state | dtls_client: dtls_client}
-
-    {:ok, dtls_fingerprint} = ExDTLS.get_cert_fingerprint(state.dtls_client)
-
     offer =
       %ExSDP{ExSDP.new() | timing: %ExSDP.Timing{start_time: 0, stop_time: 0}}
       # we support trickle ICE only
@@ -161,7 +154,7 @@ defmodule ExWebRTC.PeerConnection do
         ice_ufrag: ice_ufrag,
         ice_pwd: ice_pwd,
         ice_options: "trickle",
-        fingerprint: {:sha256, Utils.hex_dump(dtls_fingerprint)},
+        fingerprint: {:sha256, Utils.hex_dump(state.dtls_transport.fingerprint)},
         setup: :actpass,
         rtcp: true
       ]
@@ -205,11 +198,6 @@ defmodule ExWebRTC.PeerConnection do
     {:offer, remote_offer} = state.pending_remote_desc
 
     {:ok, ice_ufrag, ice_pwd} = ICEAgent.get_local_credentials(state.ice_agent)
-    # TODO this will fail on subsequent calls
-    {:ok, dtls_client} = ExDTLS.start_link(client_mode: true, dtls_srtp: true)
-    state = %__MODULE__{state | dtls_client: dtls_client}
-
-    {:ok, dtls_fingerprint} = ExDTLS.get_cert_fingerprint(state.dtls_client)
 
     answer =
       %ExSDP{ExSDP.new() | timing: %ExSDP.Timing{start_time: 0, stop_time: 0}}
@@ -221,7 +209,7 @@ defmodule ExWebRTC.PeerConnection do
         ice_ufrag: ice_ufrag,
         ice_pwd: ice_pwd,
         ice_options: "trickle",
-        fingerprint: {:sha256, Utils.hex_dump(dtls_fingerprint)},
+        fingerprint: {:sha256, Utils.hex_dump(state.dtls_transport.fingerprint)},
         setup: :active
       ]
 
@@ -331,20 +319,8 @@ defmodule ExWebRTC.PeerConnection do
 
   @impl true
   def handle_info({:ex_ice, _from, :connected}, state) do
-    state =
-      if state.initiate_dtls do
-        start_dtls_handshake(state)
-        %__MODULE__{state | initiate_dtls: false}
-      else
-        state
-      end
-
-    if state.dtls_buffered_packets do
-      Logger.debug("Sending buffered DTLS packets")
-      ICEAgent.send_data(state.ice_agent, state.dtls_buffered_packets)
-    end
-
-    {:noreply, %__MODULE__{state | ice_state: :connected, dtls_buffered_packets: nil}}
+    dtls = DTLSTransport.update_ice_state(state.dtls_transport, :connected)
+    {:noreply, %__MODULE__{state | dtls_transport: dtls, ice_state: :connected}}
   end
 
   @impl true
@@ -362,50 +338,16 @@ defmodule ExWebRTC.PeerConnection do
   end
 
   @impl true
-  def handle_info({:ex_ice, _from, {:data, data}}, %{dtls_finished: false} = state) do
-    case ExDTLS.process(state.dtls_client, data) do
-      {:handshake_packets, packets} when state.ice_state in [:connected, :completed] ->
-        :ok = ICEAgent.send_data(state.ice_agent, packets)
-        {:noreply, state}
-
-      {:handshake_packets, packets} ->
-        Logger.debug("""
-        Generated local DTLS packets but ICE is not in the connected or completed state yet.
-        We will send those packets once ICE is ready.
-        """)
-
-        {:noreply, %__MODULE__{state | dtls_buffered_packets: packets}}
-
-      {:handshake_finished, _keying_material, packets} ->
-        Logger.debug("DTLS handshake finished")
-        ICEAgent.send_data(state.ice_agent, packets)
-        {:noreply, %__MODULE__{state | dtls_finished: true}}
-
-      {:handshake_finished, _keying_material} ->
-        Logger.debug("DTLS handshake finished")
-        {:noreply, %__MODULE__{state | dtls_finished: true}}
-
-      :handshake_want_read ->
-        {:noreply, state}
-    end
-  end
-
-  @impl true
-  def handle_info({:ex_dtls, _from, {:retransmit, packets}}, state)
-      when state.ice_state in [:connected, :completed] do
-    ICEAgent.send_data(state.ice_agent, packets)
-    {:noreply, state}
+  def handle_info({:ex_ice, _from, {:data, data}}, state)
+      when not state.dtls_transport.finished do
+    dtls = DTLSTransport.process_data(state.dtls_transport, data)
+    {:noreply, %__MODULE__{state | dtls_transport: dtls}}
   end
 
   @impl true
-  def handle_info(
-        {:ex_dtls, _from, {:retransmit, packets}},
-        %{dtls_buffered_packets: packets} = state
-      ) do
-    # we got DTLS packets from the other side but
-    # we haven't established ICE connection yet so
-    # packets to retransmit have to be the same as dtls_buffered_packets
-    {:noreply, state}
+  def handle_info({:ex_dtls, _from, msg}, state) do
+    dtls = DTLSTransport.handle_info(state.dtls_transport, msg)
+    {:noreply, %__MODULE__{state | dtls_transport: dtls}}
   end
 
   @impl true
@@ -418,15 +360,15 @@ defmodule ExWebRTC.PeerConnection do
     new_transceivers = update_local_transceivers(type, state.transceivers, sdp)
     state = set_description(:local, type, sdp, state)
 
-    state =
+    dtls =
       if type == :answer do
         {:setup, setup} = ExSDP.Media.get_attribute(hd(sdp.media), :setup)
-        start_dtls(setup, state)
+        DTLSTransport.start(state.dtls_transport, setup)
       else
-        state
+        state.dtls_transport
       end
 
-    {:ok, %{state | transceivers: new_transceivers}}
+    {:ok, %{state | transceivers: new_transceivers, dtls_transport: dtls}}
   end
 
   defp update_local_transceivers(:offer, transceivers, _sdp) do
@@ -462,15 +404,22 @@ defmodule ExWebRTC.PeerConnection do
 
       state = set_description(:remote, type, sdp, state)
 
-      state =
+      dtls =
         if type == :answer do
           {:setup, setup} = ExSDP.Media.get_attribute(hd(sdp.media), :setup)
-          start_dtls(setup, state)
+
+          setup =
+            case setup do
+              :active -> :passive
+              :passive -> :active
+            end
+
+          DTLSTransport.start(state.dtls_transport, setup)
         else
-          state
+          state.dtls_transport
         end
 
-      {:ok, %{state | transceivers: new_transceivers}}
+      {:ok, %{state | transceivers: new_transceivers, dtls_transport: dtls}}
     else
       error -> error
     end
@@ -499,24 +448,6 @@ defmodule ExWebRTC.PeerConnection do
     new_transceivers
   end
 
-  defp start_dtls(:active, %{ice_state: :connected} = state) do
-    start_dtls_handshake(state)
-    state
-  end
-
-  defp start_dtls(:active, state) do
-    %__MODULE__{state | initiate_dtls: true}
-  end
-
-  defp start_dtls(_setup, state) do
-    state
-  end
-
-  defp start_dtls_handshake(state) do
-    {:ok, packets} = ExDTLS.do_handshake(state.dtls_client)
-    :ok = ICEAgent.send_data(state.ice_agent, packets)
-  end
-
   defp find_next_mid(state) do
     # next mid must be unique, it's acomplished by looking for values
     # greater than any mid in remote description or our own transceivers