Feature/url extractor

lib/radiator/application.ex

@@ -5,8 +5,9 @@ defmodule Radiator.Application do
 
   use Application
 
-  alias Radiator.Outline.EventConsumer
+  alias Radiator.Outline.CommandProcessor
   alias Radiator.Outline.EventProducer
+  alias Radiator.Outline.NodeChangeListener
 
   @impl true
   def start(_type, _args) do
@@ -22,7 +23,8 @@ defmodule Radiator.Application do
       # Start to serve requests, typically the last entry
       RadiatorWeb.Endpoint,
       {EventProducer, name: EventProducer},
-      {EventConsumer, name: EventConsumer, subscribe_to: [{EventProducer, max_demand: 1}]}
+      {CommandProcessor, name: CommandProcessor, subscribe_to: [{EventProducer, max_demand: 1}]},
+      {NodeChangeListener, name: NodeChangeListener}
     ]
 
     # See https://hexdocs.pm/elixir/Supervisor.html

lib/radiator/outline/event_consumer.ex → lib/radiator/outline/command_processor.ex

@@ -1,4 +1,4 @@
-defmodule Radiator.Outline.EventConsumer do
+defmodule Radiator.Outline.CommandProcessor do
   @moduledoc false
 
   use GenStage
@@ -78,6 +78,7 @@ defmodule Radiator.Outline.EventConsumer do
 
         %NodeDeletedEvent{
           node_id: node_id,
+          episode_id: node.episode_id,
           uuid: command.event_id,
           user_id: command.user_id,
           children: result.children,
@@ -95,7 +96,8 @@ defmodule Radiator.Outline.EventConsumer do
       node: node,
       uuid: command.event_id,
       user_id: command.user_id,
-      next_id: next_id
+      next_id: next_id,
+      episode_id: node.episode_id
     }
     |> EventStore.persist_event()
     |> Dispatch.broadcast()
@@ -118,7 +120,8 @@ defmodule Radiator.Outline.EventConsumer do
       prev_id: command.prev_id,
       user_id: command.user_id,
       uuid: command.event_id,
-      next_id: result.next_id
+      next_id: result.next_id,
+      episode_id: node.episode_id
     }
     |> EventStore.persist_event()
     |> Dispatch.broadcast()
@@ -136,7 +139,8 @@ defmodule Radiator.Outline.EventConsumer do
       node_id: node.uuid,
       content: node.content,
       user_id: command.user_id,
-      uuid: command.event_id
+      uuid: command.event_id,
+      episode_id: node.episode_id
     }
     |> EventStore.persist_event()
     |> Dispatch.broadcast()

lib/radiator/outline/dispatch.ex

@@ -27,7 +27,7 @@ defmodule Radiator.Outline.Dispatch do
     |> EventProducer.enqueue()
   end
 
-  def subscribe(_episode_id) do
+  def subscribe do
     Phoenix.PubSub.subscribe(Radiator.PubSub, "events")
   end
 

lib/radiator/outline/event.ex

@@ -10,8 +10,6 @@ defmodule Radiator.Outline.Event do
     NodeMovedEvent
   }
 
-  alias Radiator.Outline.NodeRepository
-
   def payload(%NodeInsertedEvent{} = event) do
     %{
       node_id: event.node.uuid,
@@ -27,7 +25,12 @@ defmodule Radiator.Outline.Event do
   end
 
   def payload(%NodeDeletedEvent{} = event) do
-    %{node_id: event.node_id, children: event.children, next_id: event.next_id}
+    %{
+      node_id: event.node_id,
+      episode_id: event.episode_id,
+      children: event.children,
+      next_id: event.next_id
+    }
   end
 
   def payload(%NodeMovedEvent{} = event) do
@@ -42,26 +45,12 @@ defmodule Radiator.Outline.Event do
   end
 
   def event_type(%NodeInsertedEvent{} = _event), do: "NodeInsertedEvent"
-
   def event_type(%NodeContentChangedEvent{} = _event), do: "NodeContentChangedEvent"
-
   def event_type(%NodeDeletedEvent{} = _event), do: "NodeDeletedEvent"
-
   def event_type(%NodeMovedEvent{} = _event), do: "NodeMovedEvent"
 
-  def episode_id(%NodeInsertedEvent{} = event) do
-    event.node.episode_id
-  end
-
-  def episode_id(%NodeContentChangedEvent{} = event) do
-    NodeRepository.get_node(event.node_id).episode_id
-  end
-
-  def episode_id(%NodeDeletedEvent{} = event) do
-    NodeRepository.get_node(event.next_id).episode_id
-  end
-
-  def episode_id(%NodeMovedEvent{} = event) do
-    NodeRepository.get_node(event.node_id).episode_id
-  end
+  def episode_id(%NodeInsertedEvent{episode_id: episode_id}), do: episode_id
+  def episode_id(%NodeContentChangedEvent{episode_id: episode_id}), do: episode_id
+  def episode_id(%NodeDeletedEvent{episode_id: episode_id}), do: episode_id
+  def episode_id(%NodeMovedEvent{episode_id: episode_id}), do: episode_id
 end

lib/radiator/outline/event/node_content_changed_event.ex

@@ -1,5 +1,5 @@
 defmodule Radiator.Outline.Event.NodeContentChangedEvent do
   @moduledoc false
 
-  defstruct [:uuid, :node_id, :content, :user_id]
+  defstruct [:uuid, :node_id, :content, :user_id, :episode_id]
 end

lib/radiator/outline/event/node_deleted_event.ex

@@ -1,4 +1,4 @@
 defmodule Radiator.Outline.Event.NodeDeletedEvent do
   @moduledoc false
-  defstruct [:uuid, :node_id, :user_id, :children, :next_id]
+  defstruct [:uuid, :node_id, :user_id, :children, :next_id, :episode_id]
 end

lib/radiator/outline/event/node_inserted_event.ex

@@ -1,5 +1,5 @@
 defmodule Radiator.Outline.Event.NodeInsertedEvent do
   @moduledoc false
 
-  defstruct [:uuid, :node, :user_id, :next_id]
+  defstruct [:uuid, :node, :user_id, :next_id, :episode_id]
 end

lib/radiator/outline/event/node_moved_event.ex

@@ -8,6 +8,7 @@ defmodule Radiator.Outline.Event.NodeMovedEvent do
     :user_id,
     :old_prev_id,
     :old_next_id,
-    :next_id
+    :next_id,
+    :episode_id
   ]
 end

lib/radiator/outline/node_change_listener.ex

@@ -0,0 +1,63 @@
+defmodule Radiator.Outline.NodeChangeListener do
+  @moduledoc """
+  Genserver that listens to change events and starts jobs
+  It is an eventconsumer that listens to changes in the outline and starts workers
+  """
+  use GenServer
+
+  alias Radiator.Outline.Event.{
+    NodeContentChangedEvent,
+    NodeDeletedEvent,
+    NodeInsertedEvent,
+    NodeMovedEvent
+  }
+
+  alias Radiator.Outline.Dispatch
+  alias Radiator.UrlExtractor
+
+  require Logger
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, :ok, [])
+  end
+
+  def init(_) do
+    Dispatch.subscribe()
+    {:ok, []}
+  end
+
+  def handle_info(%NodeContentChangedEvent{node_id: node_id, content: content}, state) do
+    scan_content_for_urls(node_id, content)
+    {:noreply, state}
+  end
+
+  def handle_info(%NodeInsertedEvent{} = _event, state) do
+    {:noreply, state}
+  end
+
+  def handle_info(%NodeMovedEvent{} = _event, state) do
+    {:noreply, state}
+  end
+
+  def handle_info(%NodeDeletedEvent{} = _event, state) do
+    {:noreply, state}
+  end
+
+  def handle_info(_reference, state) do
+    Logger.warning("Unknown event type")
+    {:noreply, state}
+  end
+
+  defp scan_content_for_urls(_node_id, nil), do: nil
+
+  defp scan_content_for_urls(_node_id, content) do
+    Task.async(fn ->
+      result = UrlExtractor.extract_urls(content)
+      Logger.debug("Extracted #{Enum.count(result)} Urls!!")
+
+      Enum.each(result, fn info ->
+        Logger.debug("URL: #{info.url}")
+      end)
+    end)
+  end
+end

lib/radiator/url_extractor.ex

@@ -0,0 +1,26 @@
+defmodule Radiator.UrlExtractor do
+  @moduledoc """
+  extract urls
+  """
+  @url_regex ~r/((([A-Za-z]{3,9}:(?:\/\/)?)(?:[-;:&=\+\$,\w]+@)?[A-Za-z0-9.-]+|(?:www.|[-;:&=\+\$,\w]+@)[A-Za-z0-9.-]+)((?:\/[\+~%\/.\w-_]*)?\??(?:[-\+=&;%@.\w_]*)#?(?:[\w]*))?)/
+
+  def extract_urls(text) do
+    text
+    |> extract_url_positions
+    |> Enum.map(fn {start_bytes, size_bytes} ->
+      %{
+        start_bytes: start_bytes,
+        size_bytes: size_bytes,
+        url: String.byte_slice(text, start_bytes, size_bytes)
+      }
+    end)
+  end
+
+  # should return two URLs that we can parse/scrape later
+  # @return [{Integer.t, Integer.t}] list of positions of URLs in the text
+  def extract_url_positions(text) do
+    @url_regex
+    |> Regex.scan(text, return: :index)
+    |> Enum.map(&hd/1)
+  end
+end

lib/radiator_web/live/episode_live/index.ex

@@ -35,8 +35,8 @@ defmodule RadiatorWeb.EpisodeLive.Index do
   def handle_params(params, _uri, socket) do
     episode = get_selected_episode(params)
 
-    if connected?(socket) and episode do
-      Dispatch.subscribe(episode.id)
+    if connected?(socket) do
+      Dispatch.subscribe()
     end
 
     socket

test/radiator/outline/event_consumer_test.exs → test/radiator/outline/command_processor_test.exs

@@ -1,10 +1,10 @@
-defmodule Radiator.Outline.EventConsumerTest do
+defmodule Radiator.Outline.CommandProcessorTest do
   alias Radiator.Outline.NodeRepository
   use Radiator.DataCase
 
   alias Radiator.AccountsFixtures
   alias Radiator.EventStore
-  alias Radiator.Outline.{Command, Dispatch, EventConsumer, EventProducer, NodeRepository}
+  alias Radiator.Outline.{Command, CommandProcessor, Dispatch, EventProducer, NodeRepository}
   alias Radiator.Outline.Command.InsertNodeCommand
   alias Radiator.Outline.Event.NodeInsertedEvent
   alias Radiator.PodcastFixtures
@@ -21,7 +21,7 @@ defmodule Radiator.Outline.EventConsumerTest do
 
       num_nodes = NodeRepository.count_nodes_by_episode(episode.id)
       command = Command.build("insert_node", attributes, user.id, event_id)
-      EventConsumer.handle_events([command], 0, nil)
+      CommandProcessor.handle_events([command], 0, nil)
 
       # assert a node has been created
       assert num_nodes + 1 == NodeRepository.count_nodes_by_episode(episode.id)
@@ -41,7 +41,7 @@ defmodule Radiator.Outline.EventConsumerTest do
       }
 
       command = Command.build("insert_node", attributes, user.id, event_id)
-      EventConsumer.handle_events([command], 0, nil)
+      CommandProcessor.handle_events([command], 0, nil)
       event = EventStore.list_event_data() |> hd()
 
       assert event.event_type == "NodeInsertedEvent"
@@ -62,11 +62,11 @@ defmodule Radiator.Outline.EventConsumerTest do
         }
       }
 
-      Dispatch.subscribe(episode.id)
+      Dispatch.subscribe()
       EventProducer.enqueue(producer, command)
 
       start_supervised!(
-        {EventConsumer, name: TestEventConsumer, subscribe_to: [{producer, max_demand: 1}]}
+        {CommandProcessor, name: TestCommandProcessor, subscribe_to: [{producer, max_demand: 1}]}
       )
 
       assert_receive(%NodeInsertedEvent{}, 1000)

test/radiator/url_extractor_test.exs

@@ -0,0 +1,31 @@
+defmodule Radiator.UrlExtractorTest do
+  use Radiator.DataCase
+
+  alias Radiator.UrlExtractor
+
+  describe "extract_url_positions/1" do
+    test "recognizes an URL" do
+      assert UrlExtractor.extract_url_positions("https://www.google.com") ==
+               [{0, 22}]
+    end
+
+    test "recognizes an URL with text behind" do
+      assert UrlExtractor.extract_url_positions(
+               "https://www.youtube.com/watch?v=kBU4v609DOU&t=1268s foo"
+             ) ==
+               [{0, 51}]
+    end
+
+    test "recognizes an URL with text before" do
+      assert UrlExtractor.extract_url_positions("bar https://github.com/podlove/radiator") ==
+               [{4, 35}]
+    end
+
+    test "extracts urls in text" do
+      assert UrlExtractor.extract_url_positions(
+               "das ist eine super URL: https://www.freecodecamp.org/news/how-to-write-a-regular-expression-for-a-url/ und das auch https://hexdocs.pm/elixir/Regex.html#scan/3"
+             ) ==
+               [{24, 78}, {116, 41}]
+    end
+  end
+end