diff --git a/.env.sample b/.env.sample deleted file mode 100644 index c774b613..00000000 --- a/.env.sample +++ /dev/null @@ -1,3 +0,0 @@ -POSTGRES_HOST=postgres -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres diff --git a/.github/workflows/lint_and_test.yml b/.github/workflows/lint_and_test.yml index 09f9595c..8983a879 100644 --- a/.github/workflows/lint_and_test.yml +++ b/.github/workflows/lint_and_test.yml @@ -24,9 +24,6 @@ jobs: - name: Create and populate ENV file run: | echo MIX_ENV=test >> .env - echo POSTGRES_HOST=postgres >> .env - echo POSTGRES_USER=postgres >> .env - echo POSTGRES_PASSWORD=postgres >> .env - name: Pull prebuilt images run: docker compose pull diff --git a/.gitignore b/.gitignore index f0a1e888..23522160 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,7 @@ npm-debug.log .DS_Store scratchpad.md /scratchpad/ + +# Database files +*.db +*.db-* diff --git a/Dockerfile b/Dockerfile index e987099e..84b1fafe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM elixir:latest # Install debian packages RUN apt-get update -qq -RUN apt-get install -y inotify-tools postgresql-client ffmpeg \ +RUN apt-get install -y inotify-tools ffmpeg \ python3 python3-pip python3-setuptools python3-wheel python3-dev # Install nodejs diff --git a/config/config.exs b/config/config.exs index 91b39206..1936df3e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -29,6 +29,7 @@ config :pinchflat, PinchflatWeb.Endpoint, live_view: [signing_salt: "/t5878kO"] config :pinchflat, Oban, + engine: Oban.Engines.Lite, repo: Pinchflat.Repo, # Keep old jobs for 30 days for display in the UI plugins: [{Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60}], diff --git a/config/dev.exs b/config/dev.exs index ccf1d708..2dda9d9b 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -6,13 +6,9 @@ config :pinchflat, # Configure your database config :pinchflat, Pinchflat.Repo, - username: System.get_env("POSTGRES_USER"), - password: System.get_env("POSTGRES_PASSWORD"), - hostname: System.get_env("POSTGRES_HOST"), - database: "pinchflat_dev", - stacktrace: true, + database: Path.expand("../priv/repo/pinchflat_dev.db", Path.dirname(__ENV__.file)), show_sensitive_data_on_connection_error: true, - pool_size: 10 + pool_size: 5 # For development, we disable any cache and enable # debugging and code reloading. diff --git a/config/runtime.exs b/config/runtime.exs index e12f2270..af11e69e 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -21,20 +21,16 @@ if System.get_env("PHX_SERVER") do end if config_env() == :prod do - database_url = - System.get_env("DATABASE_URL") || + database_path = + System.get_env("DATABASE_PATH") || raise """ - environment variable DATABASE_URL is missing. - For example: ecto://USER:PASS@HOST/DATABASE + environment variable DATABASE_PATH is missing. + For example: /etc/pinchflat/pinchflat.db """ - maybe_ipv6 = if System.get_env("ECTO_IPV6") in ~w(true 1), do: [:inet6], else: [] - config :pinchflat, Pinchflat.Repo, - # ssl: true, - url: database_url, - pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10"), - socket_options: maybe_ipv6 + database: database_path, + pool_size: String.to_integer(System.get_env("POOL_SIZE") || "5") # The secret key base is used to sign/encrypt cookies and other secrets. # A default value is used in config/dev.exs and config/test.exs but you diff --git a/config/test.exs b/config/test.exs index 94e19197..e51fb696 100644 --- a/config/test.exs +++ b/config/test.exs @@ -14,12 +14,9 @@ config :pinchflat, Oban, testing: :manual # to provide built-in test partitioning in CI environment. # Run `mix help test` for more information. config :pinchflat, Pinchflat.Repo, - username: System.get_env("POSTGRES_USER"), - password: System.get_env("POSTGRES_PASSWORD"), - hostname: System.get_env("POSTGRES_HOST"), - database: "pinchflat_test#{System.get_env("MIX_TEST_PARTITION")}", - pool: Ecto.Adapters.SQL.Sandbox, - pool_size: 10 + database: Path.expand("../priv/repo/pinchflat_test.db", Path.dirname(__ENV__.file)), + pool_size: 5, + pool: Ecto.Adapters.SQL.Sandbox # We don't run a server during test. If one is required, # you can enable the server option below. diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 432e2335..19e7aae2 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -1,11 +1,5 @@ version: '3' services: - postgres: - image: 'postgres:16-alpine' - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - phx: build: . volumes: @@ -13,4 +7,3 @@ services: ports: - '4008:4008' command: tail -F /dev/null - env_file: .env diff --git a/docker-compose.yml b/docker-compose.yml index 34d2e2a0..2f89a504 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,22 +1,12 @@ version: '3' services: - postgres: - image: 'postgres:16-alpine' - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - phx: build: . volumes: - '.:/app' ports: - '4008:4008' - depends_on: - - postgres command: - ./docker-run.sh stdin_open: true tty: true - env_file: - - .env diff --git a/docker-run.sh b/docker-run.sh index 4496d153..ddae66f2 100755 --- a/docker-run.sh +++ b/docker-run.sh @@ -10,15 +10,6 @@ echo "\nInstalling JS..." cd assets && yarn install cd .. -# Wait for Postgres to become available. -export PGPASSWORD=$(echo $POSTGRES_PASSWORD) -until psql -h postgres -U $POSTGRES_USER -c '\q' 2>/dev/null; do - echo >&2 "Postgres is unavailable - sleeping" - sleep 1 -done - -echo "\nPostgres is available: continuing with database setup..." - # Potentially Set up the database mix ecto.create mix ecto.migrate diff --git a/lib/pinchflat/media.ex b/lib/pinchflat/media.ex index 14ae494f..cdd3fd62 100644 --- a/lib/pinchflat/media.ex +++ b/lib/pinchflat/media.ex @@ -51,31 +51,29 @@ defmodule Pinchflat.Media do Returns a list of media_items that match the search term. Adds a `matching_search_term` virtual field to the result set. + Has explit handling for blank search terms because SQLite doesn't like empty MATCH clauses. + Returns [%MediaItem{}, ...]. """ - def search(search_term, opts \\ []) do + def search(_search_term, _opts \\ []) + def search("", _opts), do: [] + def search(nil, _opts), do: [] + + def search(search_term, opts) do limit = Keyword.get(opts, :limit, 50) from(mi in MediaItem, - where: fragment("searchable @@ websearch_to_tsquery(?)", ^search_term), + join: mi_search_index in assoc(mi, :media_items_search_index), + where: fragment("media_items_search_index MATCH ?", ^search_term), select_merge: %{ matching_search_term: - fragment( - """ - ts_headline( - 'english', - CONCAT(title, ' ', description), - websearch_to_tsquery(?), - 'StartSel=[PF_HIGHLIGHT],StopSel=[/PF_HIGHLIGHT]' - ) - """, - ^search_term - ) - }, - order_by: { - :desc, - fragment("ts_rank_cd(searchable, websearch_to_tsquery(?), 0)", ^search_term) + fragment(""" + snippet(media_items_search_index, 0, '[PF_HIGHLIGHT]', '[/PF_HIGHLIGHT]', '...', 20) || + ' ' || + snippet(media_items_search_index, 1, '[PF_HIGHLIGHT]', '[/PF_HIGHLIGHT]', '...', 20) + """) }, + order_by: [desc: fragment("rank")], limit: ^limit ) |> Repo.all() @@ -174,7 +172,10 @@ defmodule Pinchflat.Media do Enum.reduce(mapped_struct, dynamic(true), fn attr, dynamic -> case {attr, media_profile} do {{:shorts_behaviour, :only}, %{livestream_behaviour: :only}} -> - dynamic([mi], ^dynamic and (mi.livestream == true or fragment("? ILIKE ?", mi.original_url, "%/shorts/%"))) + dynamic( + [mi], + ^dynamic and (mi.livestream == true or fragment("LOWER(?) LIKE LOWER(?)", mi.original_url, "%/shorts/%")) + ) # Technically redundant, but makes the other clauses easier to parse # (redundant because this condition is the same as the condition above, just flipped) @@ -183,7 +184,7 @@ defmodule Pinchflat.Media do {{:shorts_behaviour, :only}, _} -> # return records with /shorts/ in the original_url - dynamic([mi], ^dynamic and fragment("? ILIKE ?", mi.original_url, "%/shorts/%")) + dynamic([mi], ^dynamic and fragment("LOWER(?) LIKE LOWER(?)", mi.original_url, "%/shorts/%")) {{:livestream_behaviour, :only}, _} -> # return records with livestream: true @@ -191,7 +192,7 @@ defmodule Pinchflat.Media do {{:shorts_behaviour, :exclude}, %{livestream_behaviour: lb}} when lb != :only -> # return records without /shorts/ in the original_url - dynamic([mi], ^dynamic and fragment("? NOT ILIKE ?", mi.original_url, "%/shorts/%")) + dynamic([mi], ^dynamic and fragment("LOWER(?) NOT LIKE LOWER(?)", mi.original_url, "%/shorts/%")) {{:livestream_behaviour, :exclude}, %{shorts_behaviour: sb}} when sb != :only -> # return records with livestream: false diff --git a/lib/pinchflat/media/media_item.ex b/lib/pinchflat/media/media_item.ex index b43a1a2f..c9c99d6b 100644 --- a/lib/pinchflat/media/media_item.ex +++ b/lib/pinchflat/media/media_item.ex @@ -9,6 +9,7 @@ defmodule Pinchflat.Media.MediaItem do alias Pinchflat.Tasks.Task alias Pinchflat.MediaSource.Source alias Pinchflat.Media.MediaMetadata + alias Pinchflat.Media.MediaItemSearchIndex @allowed_fields ~w( title @@ -46,6 +47,7 @@ defmodule Pinchflat.Media.MediaItem do belongs_to :source, Source has_one :metadata, MediaMetadata, on_replace: :update + has_one :media_items_search_index, MediaItemSearchIndex, foreign_key: :id has_many :tasks, Task diff --git a/lib/pinchflat/media/media_items_search_index.ex b/lib/pinchflat/media/media_items_search_index.ex new file mode 100644 index 00000000..c55d7877 --- /dev/null +++ b/lib/pinchflat/media/media_items_search_index.ex @@ -0,0 +1,16 @@ +defmodule Pinchflat.Media.MediaItemSearchIndex do + @moduledoc """ + The MediaItem fts5 search index. Not made to be directly interacted with, + but I figured it'd be better to have it in-app so it's not a mystery. + """ + + use Ecto.Schema + + @primary_key {:id, :id, autogenerate: true, source: :rowid} + schema "media_items_search_index" do + field :title, :string + field :description, :string + + field :rank, :float, virtual: true + end +end diff --git a/lib/pinchflat/repo.ex b/lib/pinchflat/repo.ex index 9751f758..1ade75a8 100644 --- a/lib/pinchflat/repo.ex +++ b/lib/pinchflat/repo.ex @@ -1,7 +1,7 @@ defmodule Pinchflat.Repo do use Ecto.Repo, otp_app: :pinchflat, - adapter: Ecto.Adapters.Postgres + adapter: Ecto.Adapters.SQLite3 @doc """ It's not immediately obvious if an Oban job qualifies as unique, so this method diff --git a/mix.exs b/mix.exs index d89ada40..13c726a9 100644 --- a/mix.exs +++ b/mix.exs @@ -35,7 +35,7 @@ defmodule Pinchflat.MixProject do {:phoenix, "~> 1.7.10"}, {:phoenix_ecto, "~> 4.4"}, {:ecto_sql, "~> 3.10"}, - {:postgrex, ">= 0.0.0"}, + {:ecto_sqlite3, ">= 0.0.0"}, {:phoenix_html, "~> 3.3"}, {:phoenix_live_reload, "~> 1.2", only: :dev}, {:phoenix_live_view, "~> 0.20.1"}, diff --git a/mix.lock b/mix.lock index 157a94f8..8f11b12e 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,7 @@ %{ "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "castore": {:hex, :castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [:mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"}, + "cc_precompiler": {:hex, :cc_precompiler, "0.1.9", "e8d3364f310da6ce6463c3dd20cf90ae7bbecbf6c5203b98bf9b48035592649b", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "9dcab3d0f3038621f1601f13539e7a9ee99843862e66ad62827b0c42b2f58a54"}, "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"}, @@ -10,8 +11,11 @@ "dns_cluster": {:hex, :dns_cluster, "0.1.2", "3eb5be824c7888dadf9781018e1a5f1d3d1113b333c50bce90fb1b83df1015f2", [:mix], [], "hexpm", "7494272040f847637bbdb01bcdf4b871e82daf09b813e7d3cb3b84f112c6f2f8"}, "ecto": {:hex, :ecto, "3.11.1", "4b4972b717e7ca83d30121b12998f5fcdc62ba0ed4f20fd390f16f3270d85c3e", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebd3d3772cd0dfcd8d772659e41ed527c28b2a8bde4b00fe03e0463da0f1983b"}, "ecto_sql": {:hex, :ecto_sql, "3.11.1", "e9abf28ae27ef3916b43545f9578b4750956ccea444853606472089e7d169470", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ce14063ab3514424276e7e360108ad6c2308f6d88164a076aac8a387e1fea634"}, + "ecto_sqlite3": {:hex, :ecto_sqlite3, "0.15.1", "40f2fbd9e246455f8c42e7e0a77009ef806caa1b3ce6f717b2a0a80e8432fcfd", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.11", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:exqlite, "~> 0.19", [hex: :exqlite, repo: "hexpm", optional: false]}], "hexpm", "28b16e177123c688948357176662bf9ff9084daddf950ef5b6baf3ee93707064"}, + "elixir_make": {:hex, :elixir_make, "0.7.8", "505026f266552ee5aabca0b9f9c229cbb496c689537c9f922f3eb5431157efc7", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "7a71945b913d37ea89b06966e1342c85cfe549b15e6d6d081e8081c493062c07"}, "esbuild": {:hex, :esbuild, "0.8.1", "0cbf919f0eccb136d2eeef0df49c4acf55336de864e63594adcea3814f3edf41", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "25fc876a67c13cb0a776e7b5d7974851556baeda2085296c14ab48555ea7560f"}, "expo": {:hex, :expo, "0.5.1", "249e826a897cac48f591deba863b26c16682b43711dd15ee86b92f25eafd96d9", [:mix], [], "hexpm", "68a4233b0658a3d12ee00d27d37d856b1ba48607e7ce20fd376958d0ba6ce92b"}, + "exqlite": {:hex, :exqlite, "0.19.0", "0f3ee29e35bed38552dd0ed59600aa81c78f867f5b5ff0e17d330148e0465483", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "55a8fbb0443f03d4a256e3458bd1203eff5037a6624b76460eaaa9080f462b06"}, "faker": {:hex, :faker, "0.17.0", "671019d0652f63aefd8723b72167ecdb284baf7d47ad3a82a15e9b8a6df5d1fa", [:mix], [], "hexpm", "a7d4ad84a93fd25c5f5303510753789fc2433ff241bf3b4144d3f6f291658a6a"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "finch": {:hex, :finch, "0.17.0", "17d06e1d44d891d20dbd437335eebe844e2426a0cd7e3a3e220b461127c73f70", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8d014a661bb6a437263d4b5abf0bcbd3cf0deb26b1e8596f2a271d22e48934c7"}, diff --git a/priv/repo/migrations/20240123174417_create_channels.exs b/priv/repo/migrations/20240123174417_create_channels.exs index 4430c3dd..173dd721 100644 --- a/priv/repo/migrations/20240123174417_create_channels.exs +++ b/priv/repo/migrations/20240123174417_create_channels.exs @@ -1,17 +1,18 @@ -defmodule Pinchflat.Repo.Migrations.CreateChannels do +defmodule Pinchflat.Repo.Migrations.CreateSources do use Ecto.Migration def change do - create table(:channels) do + create table(:sources) do add :name, :string, null: false - add :channel_id, :string, null: false + add :collection_id, :string, null: false + add :collection_type, :string, null: false add :original_url, :string, null: false add :media_profile_id, references(:media_profiles, on_delete: :restrict), null: false timestamps(type: :utc_datetime) end - create index(:channels, [:media_profile_id]) - create unique_index(:channels, [:channel_id, :media_profile_id]) + create index(:sources, [:media_profile_id]) + create unique_index(:sources, [:collection_id, :media_profile_id]) end end diff --git a/priv/repo/migrations/20240125025325_create_media_items.exs b/priv/repo/migrations/20240125025325_create_media_items.exs index cfaa25fa..413083c3 100644 --- a/priv/repo/migrations/20240125025325_create_media_items.exs +++ b/priv/repo/migrations/20240125025325_create_media_items.exs @@ -6,12 +6,12 @@ defmodule Pinchflat.Repo.Migrations.CreateMediaItems do add :media_id, :string, null: false add :title, :string add :video_filepath, :string - add :channel_id, references(:channels, on_delete: :restrict), null: false + add :source_id, references(:sources, on_delete: :restrict), null: false timestamps(type: :utc_datetime) end - create index(:media_items, [:channel_id]) - create unique_index(:media_items, [:media_id, :channel_id]) + create index(:media_items, [:source_id]) + create unique_index(:media_items, [:media_id, :source_id]) end end diff --git a/priv/repo/migrations/20240125165519_add_index_frequency_to_channels.exs b/priv/repo/migrations/20240125165519_add_index_frequency_to_channels.exs index c03b25b2..4d6384e6 100644 --- a/priv/repo/migrations/20240125165519_add_index_frequency_to_channels.exs +++ b/priv/repo/migrations/20240125165519_add_index_frequency_to_channels.exs @@ -1,8 +1,8 @@ -defmodule Pinchflat.Repo.Migrations.AddIndexFrequencyToChannels do +defmodule Pinchflat.Repo.Migrations.AddIndexFrequencyToSources do use Ecto.Migration def change do - alter table(:channels) do + alter table(:sources) do add :index_frequency_minutes, :integer, default: 60 * 24, null: false end end diff --git a/priv/repo/migrations/20240125212753_create_tasks.exs b/priv/repo/migrations/20240125212753_create_tasks.exs index 7a7b5e34..4b5734d8 100644 --- a/priv/repo/migrations/20240125212753_create_tasks.exs +++ b/priv/repo/migrations/20240125212753_create_tasks.exs @@ -4,13 +4,13 @@ defmodule Pinchflat.Repo.Migrations.CreateTasks do def change do create table(:tasks) do add :job_id, references(:oban_jobs, on_delete: :delete_all), null: false - # `restrict` because we need to be sure to delete pending tasks when a channel is deleted - add :channel_id, references(:channels, on_delete: :restrict), null: true + # `restrict` because we need to be sure to delete pending tasks when a source is deleted + add :source_id, references(:sources, on_delete: :restrict), null: true timestamps(type: :utc_datetime) end create index(:tasks, [:job_id]) - create index(:tasks, [:channel_id]) + create index(:tasks, [:source_id]) end end diff --git a/priv/repo/migrations/20240129015810_create_media_metadata.exs b/priv/repo/migrations/20240129015810_create_media_metadata.exs index bd7e49f7..671b97ee 100644 --- a/priv/repo/migrations/20240129015810_create_media_metadata.exs +++ b/priv/repo/migrations/20240129015810_create_media_metadata.exs @@ -3,13 +3,12 @@ defmodule Pinchflat.Repo.Migrations.CreateMediaMetadata do def change do create table(:media_metadata) do - add :client_response, :jsonb, null: false + add :client_response, :json, null: false add :media_item_id, references(:media_items, on_delete: :delete_all), null: false timestamps(type: :utc_datetime) end create unique_index(:media_metadata, [:media_item_id]) - create index(:media_metadata, [:client_response], using: :gin) end end diff --git a/priv/repo/migrations/20240130161316_add_media_item_to_tasks.exs b/priv/repo/migrations/20240130161316_add_media_item_to_tasks.exs index bd8a9c9d..9997e8de 100644 --- a/priv/repo/migrations/20240130161316_add_media_item_to_tasks.exs +++ b/priv/repo/migrations/20240130161316_add_media_item_to_tasks.exs @@ -3,7 +3,7 @@ defmodule Pinchflat.Repo.Migrations.AddMediaItemToTasks do def change do alter table(:tasks) do - # `restrict` because we need to be sure to delete pending tasks when a channel is deleted + # `restrict` because we need to be sure to delete pending tasks when a media item is deleted add :media_item_id, references(:media_items, on_delete: :restrict), null: true end diff --git a/priv/repo/migrations/20240202040724_rename_channel_and_related_fields.exs b/priv/repo/migrations/20240202040724_rename_channel_and_related_fields.exs deleted file mode 100644 index 977756f5..00000000 --- a/priv/repo/migrations/20240202040724_rename_channel_and_related_fields.exs +++ /dev/null @@ -1,45 +0,0 @@ -defmodule Pinchflat.Repo.Migrations.RenameChannelAndRelatedFields do - use Ecto.Migration - - def change do - # Creation - create table(:sources) do - add :name, :string, null: false - add :collection_type, :string, null: false - add :collection_id, :string, null: false - add :original_url, :string, null: false - add :media_profile_id, references(:media_profiles, on_delete: :restrict), null: false - add :index_frequency_minutes, :integer, default: 60 * 24, null: false - - timestamps(type: :utc_datetime) - end - - alter table(:media_items) do - add :source_id, references(:sources, on_delete: :restrict), null: false - end - - alter table(:tasks) do - # `restrict` because we need to be sure to delete pending tasks when a source is deleted - add :source_id, references(:sources, on_delete: :restrict), null: true - end - - create index(:sources, [:media_profile_id]) - create unique_index(:sources, [:collection_id, :media_profile_id]) - - create index(:media_items, [:source_id]) - create unique_index(:media_items, [:media_id, :source_id]) - - # Deletion - drop index(:media_items, [:channel_id]) - drop unique_index(:media_items, [:media_id, :channel_id]) - - alter table(:tasks) do - # `restrict` because we need to be sure to delete pending tasks when a source is deleted - remove :channel_id, references(:channels, on_delete: :restrict), null: true - end - - alter table(:media_items) do - remove :channel_id, references(:channels, on_delete: :restrict), null: true - end - end -end diff --git a/priv/repo/migrations/20240202043042_drop_channels_table.exs b/priv/repo/migrations/20240202043042_drop_channels_table.exs deleted file mode 100644 index 6397b19e..00000000 --- a/priv/repo/migrations/20240202043042_drop_channels_table.exs +++ /dev/null @@ -1,21 +0,0 @@ -defmodule Pinchflat.Repo.Migrations.DropChannelsTable do - use Ecto.Migration - - def up do - drop table(:channels) - end - - def down do - create table(:channels) do - add :name, :string, null: false - add :channel_id, :string, null: false - add :original_url, :string, null: false - add :media_profile_id, references(:media_profiles, on_delete: :restrict), null: false - - timestamps(type: :utc_datetime) - end - - create index(:channels, [:media_profile_id]) - create unique_index(:channels, [:channel_id, :media_profile_id]) - end -end diff --git a/priv/repo/migrations/20240210051038_add_search_field_to_media_items.exs b/priv/repo/migrations/20240210051038_add_search_field_to_media_items.exs index 04de1d74..d54e63e0 100644 --- a/priv/repo/migrations/20240210051038_add_search_field_to_media_items.exs +++ b/priv/repo/migrations/20240210051038_add_search_field_to_media_items.exs @@ -2,27 +2,50 @@ defmodule Pinchflat.Repo.Migrations.AddSearchFieldToMediaItems do use Ecto.Migration def up do + # These all need to run as part of separate `execute` blocks. Do NOT ask me why. execute """ - ALTER TABLE media_items - ADD COLUMN searchable tsvector - GENERATED ALWAYS AS ( - setweight(to_tsvector('english', coalesce(title, '')), 'A') || - setweight(to_tsvector('english', coalesce(description, '')), 'B') - ) STORED; + CREATE VIRTUAL TABLE media_items_search_index USING fts5( + title, + description, + tokenize=porter + ); """ execute """ - CREATE INDEX media_items_searchable_idx ON media_items USING gin(searchable); + CREATE TRIGGER media_items_search_index_insert AFTER INSERT ON media_items BEGIN + INSERT INTO media_items_search_index( + rowid, + title, + description + ) + VALUES( + new.id, + new.title, + new.description + ); + END; + """ + + execute """ + CREATE TRIGGER media_items_search_index_update AFTER UPDATE ON media_items BEGIN + UPDATE media_items_search_index SET + title = new.title, + description = new.description + WHERE + rowid = old.id; + END; + """ + + execute """ + CREATE TRIGGER media_items_search_index_delete AFTER DELETE ON media_items BEGIN + DELETE FROM media_items_search_index WHERE rowid = old.id; + END; """ end def down do execute """ - DROP INDEX media_items_searchable_idx; + DROP TABLE media_items_search_index; """ - - alter table(:media_items) do - remove :searchable - end end end diff --git a/test/pinchflat/media_test.exs b/test/pinchflat/media_test.exs index 77d267d1..d116ddfb 100644 --- a/test/pinchflat/media_test.exs +++ b/test/pinchflat/media_test.exs @@ -212,6 +212,14 @@ defmodule Pinchflat.MediaTest do assert [_] = Media.search("dog", limit: 1) end + + test "returns an empty list when the search term is blank" do + assert [] = Media.search("") + end + + test "returns an empty list when the search term is nil" do + assert [] = Media.search(nil) + end end describe "get_media_item!/1" do diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index 3aa5fce9..ff478a0f 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -34,7 +34,6 @@ defmodule PinchflatWeb.ConnCase do setup tags do Pinchflat.DataCase.setup_sandbox(tags) - Pinchflat.DataCase.setup_temp_filepaths() {:ok, conn: Phoenix.ConnTest.build_conn()} end end diff --git a/test/support/data_case.ex b/test/support/data_case.ex index 5bf7f10b..d2177491 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -32,20 +32,6 @@ defmodule Pinchflat.DataCase do setup tags do Pinchflat.DataCase.setup_sandbox(tags) - Pinchflat.DataCase.setup_temp_filepaths() - :ok - end - - @doc """ - Sets up the temp filepaths for the media and metadata directories. - """ - def setup_temp_filepaths do - File.rm_rf!(Application.get_env(:pinchflat, :media_directory)) - File.rm_rf!(Application.get_env(:pinchflat, :metadata_directory)) - - File.mkdir_p!(Application.get_env(:pinchflat, :media_directory)) - File.mkdir_p!(Application.get_env(:pinchflat, :metadata_directory)) - :ok end