Switch to sqlite (#22)

* Updated project to use sqlite

* Edited migrations directly because I fear no man

* Updated search functions and tests to work with sqlite

* Updated build tools to remove postgres

.github/workflows/lint_and_test.yml

@@ -24,9 +24,6 @@ jobs:
       - name: Create and populate ENV file
         run: |
           echo MIX_ENV=test >> .env
-          echo POSTGRES_HOST=postgres >> .env
-          echo POSTGRES_USER=postgres >> .env
-          echo POSTGRES_PASSWORD=postgres >> .env
 
       - name: Pull prebuilt images
         run: docker compose pull

.gitignore

@@ -41,3 +41,7 @@ npm-debug.log
 .DS_Store
 scratchpad.md
 /scratchpad/
+
+# Database files
+*.db
+*.db-*

Dockerfile

@@ -3,7 +3,7 @@ FROM elixir:latest
 
 # Install debian packages
 RUN apt-get update -qq
-RUN apt-get install -y inotify-tools postgresql-client ffmpeg \
+RUN apt-get install -y inotify-tools ffmpeg \
   python3 python3-pip python3-setuptools python3-wheel python3-dev
 
 # Install nodejs

config/config.exs

@@ -29,6 +29,7 @@ config :pinchflat, PinchflatWeb.Endpoint,
   live_view: [signing_salt: "/t5878kO"]
 
 config :pinchflat, Oban,
+  engine: Oban.Engines.Lite,
   repo: Pinchflat.Repo,
   # Keep old jobs for 30 days for display in the UI
   plugins: [{Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60}],

config/dev.exs

@@ -6,13 +6,9 @@ config :pinchflat,
 
 # Configure your database
 config :pinchflat, Pinchflat.Repo,
-  username: System.get_env("POSTGRES_USER"),
-  password: System.get_env("POSTGRES_PASSWORD"),
-  hostname: System.get_env("POSTGRES_HOST"),
-  database: "pinchflat_dev",
-  stacktrace: true,
+  database: Path.expand("../priv/repo/pinchflat_dev.db", Path.dirname(__ENV__.file)),
   show_sensitive_data_on_connection_error: true,
-  pool_size: 10
+  pool_size: 5
 
 # For development, we disable any cache and enable
 # debugging and code reloading.

config/runtime.exs

@@ -21,20 +21,16 @@ if System.get_env("PHX_SERVER") do
 end
 
 if config_env() == :prod do
-  database_url =
-    System.get_env("DATABASE_URL") ||
+  database_path =
+    System.get_env("DATABASE_PATH") ||
       raise """
-      environment variable DATABASE_URL is missing.
-      For example: ecto://USER:PASS@HOST/DATABASE
+      environment variable DATABASE_PATH is missing.
+      For example: /etc/pinchflat/pinchflat.db
       """
 
-  maybe_ipv6 = if System.get_env("ECTO_IPV6") in ~w(true 1), do: [:inet6], else: []
-
   config :pinchflat, Pinchflat.Repo,
-    # ssl: true,
-    url: database_url,
-    pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10"),
-    socket_options: maybe_ipv6
+    database: database_path,
+    pool_size: String.to_integer(System.get_env("POOL_SIZE") || "5")
 
   # The secret key base is used to sign/encrypt cookies and other secrets.
   # A default value is used in config/dev.exs and config/test.exs but you

config/test.exs

@@ -14,12 +14,9 @@ config :pinchflat, Oban, testing: :manual
 # to provide built-in test partitioning in CI environment.
 # Run `mix help test` for more information.
 config :pinchflat, Pinchflat.Repo,
-  username: System.get_env("POSTGRES_USER"),
-  password: System.get_env("POSTGRES_PASSWORD"),
-  hostname: System.get_env("POSTGRES_HOST"),
-  database: "pinchflat_test#{System.get_env("MIX_TEST_PARTITION")}",
-  pool: Ecto.Adapters.SQL.Sandbox,
-  pool_size: 10
+  database: Path.expand("../priv/repo/pinchflat_test.db", Path.dirname(__ENV__.file)),
+  pool_size: 5,
+  pool: Ecto.Adapters.SQL.Sandbox
 
 # We don't run a server during test. If one is required,
 # you can enable the server option below.

docker-compose.ci.yml

@@ -1,16 +1,9 @@
 version: '3'
 services:
-  postgres:
-    image: 'postgres:16-alpine'
-    environment:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-
   phx:
     build: .
     volumes:
       - '.:/app'
     ports:
       - '4008:4008'
     command: tail -F /dev/null
-    env_file: .env

docker-compose.yml

@@ -1,22 +1,12 @@
 version: '3'
 services:
-  postgres:
-    image: 'postgres:16-alpine'
-    environment:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-
   phx:
     build: .
     volumes:
       - '.:/app'
     ports:
       - '4008:4008'
-    depends_on:
-      - postgres
     command:
       - ./docker-run.sh
     stdin_open: true
     tty: true
-    env_file:
-      - .env

docker-run.sh

@@ -10,15 +10,6 @@ echo "\nInstalling JS..."
 cd assets && yarn install
 cd ..
 
-# Wait for Postgres to become available.
-export PGPASSWORD=$(echo $POSTGRES_PASSWORD)
-until psql -h postgres -U $POSTGRES_USER -c '\q' 2>/dev/null; do
-  echo >&2 "Postgres is unavailable - sleeping"
-  sleep 1
-done
-
-echo "\nPostgres is available: continuing with database setup..."
-
 # Potentially Set up the database
 mix ecto.create
 mix ecto.migrate

lib/pinchflat/media.ex

@@ -51,31 +51,29 @@ defmodule Pinchflat.Media do
   Returns a list of media_items that match the search term. Adds a `matching_search_term`
   virtual field to the result set.
 
+  Has explit handling for blank search terms because SQLite doesn't like empty MATCH clauses.
+
   Returns [%MediaItem{}, ...].
   """
-  def search(search_term, opts \\ []) do
+  def search(_search_term, _opts \\ [])
+  def search("", _opts), do: []
+  def search(nil, _opts), do: []
+
+  def search(search_term, opts) do
     limit = Keyword.get(opts, :limit, 50)
 
     from(mi in MediaItem,
-      where: fragment("searchable @@ websearch_to_tsquery(?)", ^search_term),
+      join: mi_search_index in assoc(mi, :media_items_search_index),
+      where: fragment("media_items_search_index MATCH ?", ^search_term),
       select_merge: %{
         matching_search_term:
-          fragment(
-            """
-            ts_headline(
-              'english',
-              CONCAT(title, ' ', description),
-              websearch_to_tsquery(?),
-              'StartSel=[PF_HIGHLIGHT],StopSel=[/PF_HIGHLIGHT]'
-            )
-            """,
-            ^search_term
-          )
-      },
-      order_by: {
-        :desc,
-        fragment("ts_rank_cd(searchable, websearch_to_tsquery(?), 0)", ^search_term)
+          fragment("""
+            snippet(media_items_search_index, 0, '[PF_HIGHLIGHT]', '[/PF_HIGHLIGHT]', '...', 20) ||
+            ' ' ||
+            snippet(media_items_search_index, 1, '[PF_HIGHLIGHT]', '[/PF_HIGHLIGHT]', '...', 20)
+          """)
       },
+      order_by: [desc: fragment("rank")],
       limit: ^limit
     )
     |> Repo.all()
@@ -174,7 +172,10 @@ defmodule Pinchflat.Media do
     Enum.reduce(mapped_struct, dynamic(true), fn attr, dynamic ->
       case {attr, media_profile} do
         {{:shorts_behaviour, :only}, %{livestream_behaviour: :only}} ->
-          dynamic([mi], ^dynamic and (mi.livestream == true or fragment("? ILIKE ?", mi.original_url, "%/shorts/%")))
+          dynamic(
+            [mi],
+            ^dynamic and (mi.livestream == true or fragment("LOWER(?) LIKE LOWER(?)", mi.original_url, "%/shorts/%"))
+          )
 
         # Technically redundant, but makes the other clauses easier to parse
         # (redundant because this condition is the same as the condition above, just flipped)
@@ -183,15 +184,15 @@ defmodule Pinchflat.Media do
 
         {{:shorts_behaviour, :only}, _} ->
           # return records with /shorts/ in the original_url
-          dynamic([mi], ^dynamic and fragment("? ILIKE ?", mi.original_url, "%/shorts/%"))
+          dynamic([mi], ^dynamic and fragment("LOWER(?) LIKE LOWER(?)", mi.original_url, "%/shorts/%"))
 
         {{:livestream_behaviour, :only}, _} ->
           # return records with livestream: true
           dynamic([mi], ^dynamic and mi.livestream == true)
 
         {{:shorts_behaviour, :exclude}, %{livestream_behaviour: lb}} when lb != :only ->
           # return records without /shorts/ in the original_url
-          dynamic([mi], ^dynamic and fragment("? NOT ILIKE ?", mi.original_url, "%/shorts/%"))
+          dynamic([mi], ^dynamic and fragment("LOWER(?) NOT LIKE LOWER(?)", mi.original_url, "%/shorts/%"))
 
         {{:livestream_behaviour, :exclude}, %{shorts_behaviour: sb}} when sb != :only ->
           # return records with livestream: false

lib/pinchflat/media/media_item.ex

@@ -9,6 +9,7 @@ defmodule Pinchflat.Media.MediaItem do
   alias Pinchflat.Tasks.Task
   alias Pinchflat.MediaSource.Source
   alias Pinchflat.Media.MediaMetadata
+  alias Pinchflat.Media.MediaItemSearchIndex
 
   @allowed_fields ~w(
     title
@@ -46,6 +47,7 @@ defmodule Pinchflat.Media.MediaItem do
     belongs_to :source, Source
 
     has_one :metadata, MediaMetadata, on_replace: :update
+    has_one :media_items_search_index, MediaItemSearchIndex, foreign_key: :id
 
     has_many :tasks, Task
 

lib/pinchflat/media/media_items_search_index.ex

@@ -0,0 +1,16 @@
+defmodule Pinchflat.Media.MediaItemSearchIndex do
+  @moduledoc """
+  The MediaItem fts5 search index. Not made to be directly interacted with,
+  but I figured it'd be better to have it in-app so it's not a mystery.
+  """
+
+  use Ecto.Schema
+
+  @primary_key {:id, :id, autogenerate: true, source: :rowid}
+  schema "media_items_search_index" do
+    field :title, :string
+    field :description, :string
+
+    field :rank, :float, virtual: true
+  end
+end

lib/pinchflat/repo.ex

@@ -1,7 +1,7 @@
 defmodule Pinchflat.Repo do
   use Ecto.Repo,
     otp_app: :pinchflat,
-    adapter: Ecto.Adapters.Postgres
+    adapter: Ecto.Adapters.SQLite3
 
   @doc """
   It's not immediately obvious if an Oban job qualifies as unique, so this method

mix.exs

@@ -35,7 +35,7 @@ defmodule Pinchflat.MixProject do
       {:phoenix, "~> 1.7.10"},
       {:phoenix_ecto, "~> 4.4"},
       {:ecto_sql, "~> 3.10"},
-      {:postgrex, ">= 0.0.0"},
+      {:ecto_sqlite3, ">= 0.0.0"},
       {:phoenix_html, "~> 3.3"},
       {:phoenix_live_reload, "~> 1.2", only: :dev},
       {:phoenix_live_view, "~> 0.20.1"},

mix.lock

@@ -1,6 +1,7 @@
 %{
   "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
   "castore": {:hex, :castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [:mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"},
+  "cc_precompiler": {:hex, :cc_precompiler, "0.1.9", "e8d3364f310da6ce6463c3dd20cf90ae7bbecbf6c5203b98bf9b48035592649b", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "9dcab3d0f3038621f1601f13539e7a9ee99843862e66ad62827b0c42b2f58a54"},
   "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
   "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
@@ -10,8 +11,11 @@
   "dns_cluster": {:hex, :dns_cluster, "0.1.2", "3eb5be824c7888dadf9781018e1a5f1d3d1113b333c50bce90fb1b83df1015f2", [:mix], [], "hexpm", "7494272040f847637bbdb01bcdf4b871e82daf09b813e7d3cb3b84f112c6f2f8"},
   "ecto": {:hex, :ecto, "3.11.1", "4b4972b717e7ca83d30121b12998f5fcdc62ba0ed4f20fd390f16f3270d85c3e", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebd3d3772cd0dfcd8d772659e41ed527c28b2a8bde4b00fe03e0463da0f1983b"},
   "ecto_sql": {:hex, :ecto_sql, "3.11.1", "e9abf28ae27ef3916b43545f9578b4750956ccea444853606472089e7d169470", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ce14063ab3514424276e7e360108ad6c2308f6d88164a076aac8a387e1fea634"},
+  "ecto_sqlite3": {:hex, :ecto_sqlite3, "0.15.1", "40f2fbd9e246455f8c42e7e0a77009ef806caa1b3ce6f717b2a0a80e8432fcfd", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.11", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:exqlite, "~> 0.19", [hex: :exqlite, repo: "hexpm", optional: false]}], "hexpm", "28b16e177123c688948357176662bf9ff9084daddf950ef5b6baf3ee93707064"},
+  "elixir_make": {:hex, :elixir_make, "0.7.8", "505026f266552ee5aabca0b9f9c229cbb496c689537c9f922f3eb5431157efc7", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "7a71945b913d37ea89b06966e1342c85cfe549b15e6d6d081e8081c493062c07"},
   "esbuild": {:hex, :esbuild, "0.8.1", "0cbf919f0eccb136d2eeef0df49c4acf55336de864e63594adcea3814f3edf41", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "25fc876a67c13cb0a776e7b5d7974851556baeda2085296c14ab48555ea7560f"},
   "expo": {:hex, :expo, "0.5.1", "249e826a897cac48f591deba863b26c16682b43711dd15ee86b92f25eafd96d9", [:mix], [], "hexpm", "68a4233b0658a3d12ee00d27d37d856b1ba48607e7ce20fd376958d0ba6ce92b"},
+  "exqlite": {:hex, :exqlite, "0.19.0", "0f3ee29e35bed38552dd0ed59600aa81c78f867f5b5ff0e17d330148e0465483", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "55a8fbb0443f03d4a256e3458bd1203eff5037a6624b76460eaaa9080f462b06"},
   "faker": {:hex, :faker, "0.17.0", "671019d0652f63aefd8723b72167ecdb284baf7d47ad3a82a15e9b8a6df5d1fa", [:mix], [], "hexpm", "a7d4ad84a93fd25c5f5303510753789fc2433ff241bf3b4144d3f6f291658a6a"},
   "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
   "finch": {:hex, :finch, "0.17.0", "17d06e1d44d891d20dbd437335eebe844e2426a0cd7e3a3e220b461127c73f70", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8d014a661bb6a437263d4b5abf0bcbd3cf0deb26b1e8596f2a271d22e48934c7"},

priv/repo/migrations/20240123174417_create_channels.exs

@@ -1,17 +1,18 @@
-defmodule Pinchflat.Repo.Migrations.CreateChannels do
+defmodule Pinchflat.Repo.Migrations.CreateSources do
   use Ecto.Migration
 
   def change do
-    create table(:channels) do
+    create table(:sources) do
       add :name, :string, null: false
-      add :channel_id, :string, null: false
+      add :collection_id, :string, null: false
+      add :collection_type, :string, null: false
       add :original_url, :string, null: false
       add :media_profile_id, references(:media_profiles, on_delete: :restrict), null: false
 
       timestamps(type: :utc_datetime)
     end
 
-    create index(:channels, [:media_profile_id])
-    create unique_index(:channels, [:channel_id, :media_profile_id])
+    create index(:sources, [:media_profile_id])
+    create unique_index(:sources, [:collection_id, :media_profile_id])
   end
 end

priv/repo/migrations/20240125025325_create_media_items.exs

@@ -6,12 +6,12 @@ defmodule Pinchflat.Repo.Migrations.CreateMediaItems do
       add :media_id, :string, null: false
       add :title, :string
       add :video_filepath, :string
-      add :channel_id, references(:channels, on_delete: :restrict), null: false
+      add :source_id, references(:sources, on_delete: :restrict), null: false
 
       timestamps(type: :utc_datetime)
     end
 
-    create index(:media_items, [:channel_id])
-    create unique_index(:media_items, [:media_id, :channel_id])
+    create index(:media_items, [:source_id])
+    create unique_index(:media_items, [:media_id, :source_id])
   end
 end

priv/repo/migrations/20240125165519_add_index_frequency_to_channels.exs

@@ -1,8 +1,8 @@
-defmodule Pinchflat.Repo.Migrations.AddIndexFrequencyToChannels do
+defmodule Pinchflat.Repo.Migrations.AddIndexFrequencyToSources do
   use Ecto.Migration
 
   def change do
-    alter table(:channels) do
+    alter table(:sources) do
       add :index_frequency_minutes, :integer, default: 60 * 24, null: false
     end
   end

priv/repo/migrations/20240125212753_create_tasks.exs

@@ -4,13 +4,13 @@ defmodule Pinchflat.Repo.Migrations.CreateTasks do
   def change do
     create table(:tasks) do
       add :job_id, references(:oban_jobs, on_delete: :delete_all), null: false
-      # `restrict` because we need to be sure to delete pending tasks when a channel is deleted
-      add :channel_id, references(:channels, on_delete: :restrict), null: true
+      # `restrict` because we need to be sure to delete pending tasks when a source is deleted
+      add :source_id, references(:sources, on_delete: :restrict), null: true
 
       timestamps(type: :utc_datetime)
     end
 
     create index(:tasks, [:job_id])
-    create index(:tasks, [:channel_id])
+    create index(:tasks, [:source_id])
   end
 end

priv/repo/migrations/20240129015810_create_media_metadata.exs

@@ -3,13 +3,12 @@ defmodule Pinchflat.Repo.Migrations.CreateMediaMetadata do
 
   def change do
     create table(:media_metadata) do
-      add :client_response, :jsonb, null: false
+      add :client_response, :json, null: false
       add :media_item_id, references(:media_items, on_delete: :delete_all), null: false
 
       timestamps(type: :utc_datetime)
     end
 
     create unique_index(:media_metadata, [:media_item_id])
-    create index(:media_metadata, [:client_response], using: :gin)
   end
 end

priv/repo/migrations/20240130161316_add_media_item_to_tasks.exs

@@ -3,7 +3,7 @@ defmodule Pinchflat.Repo.Migrations.AddMediaItemToTasks do
 
   def change do
     alter table(:tasks) do
-      # `restrict` because we need to be sure to delete pending tasks when a channel is deleted
+      # `restrict` because we need to be sure to delete pending tasks when a media item is deleted
       add :media_item_id, references(:media_items, on_delete: :restrict), null: true
     end