From 8531de49fa42b1d52f79f60612accf6d1541e46a Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Thu, 15 Feb 2024 14:23:28 -0800 Subject: [PATCH 1/3] Sets media directory for prod --- config/config.exs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 91b39206..c91f31be 100644 --- a/config/config.exs +++ b/config/config.exs @@ -13,8 +13,7 @@ config :pinchflat, # Specifying backend data here makes mocking and local testing SUPER easy yt_dlp_executable: System.find_executable("yt-dlp"), yt_dlp_runner: Pinchflat.MediaClient.Backends.YtDlp.CommandRunner, - # TODO: figure this out - media_directory: :not_implemented, + media_directory: "/media", metadata_directory: Path.join([System.tmp_dir!(), "pinchflat", "metadata"]) # Configures the endpoint From 59963907769f71e7802e706c06b1e26ba7c8f83b Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Fri, 16 Feb 2024 11:36:46 -0800 Subject: [PATCH 2/3] Improved logging; Added docker files for prod --- .dockerignore | 45 +++++++ config/config.exs | 4 +- config/runtime.exs | 24 ++-- Dockerfile => dev.Dockerfile | 0 docker-compose.ci.yml | 4 +- docker-compose.yml | 6 +- docker-run.sh => docker-run.dev.sh | 0 lib/pinchflat/application.ex | 2 + .../backends/yt_dlp/command_runner.ex | 2 +- lib/pinchflat/release.ex | 28 +++++ rel/overlays/bin/docker_start | 5 + rel/overlays/bin/migrate | 3 + selfhosted.Dockerfile | 118 ++++++++++++++++++ 13 files changed, 226 insertions(+), 15 deletions(-) create mode 100644 .dockerignore rename Dockerfile => dev.Dockerfile (100%) rename docker-run.sh => docker-run.dev.sh (100%) create mode 100644 lib/pinchflat/release.ex create mode 100755 rel/overlays/bin/docker_start create mode 100755 rel/overlays/bin/migrate create mode 100644 selfhosted.Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..61a73933 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,45 @@ +# This file excludes paths from the Docker build context. +# +# By default, Docker's build context includes all files (and folders) in the +# current directory. Even if a file isn't copied into the container it is still sent to +# the Docker daemon. +# +# There are multiple reasons to exclude files from the build context: +# +# 1. Prevent nested folders from being copied into the container (ex: exclude +# /assets/node_modules when copying /assets) +# 2. Reduce the size of the build context and improve build time (ex. /build, /deps, /doc) +# 3. Avoid sending files containing sensitive information +# +# More information on using .dockerignore is available here: +# https://docs.docker.com/engine/reference/builder/#dockerignore-file + +.dockerignore + +# Ignore git, but keep git HEAD and refs to access current commit hash if needed: +# +# $ cat .git/HEAD | awk '{print ".git/"$2}' | xargs cat +# d0b8727759e1e0e7aa3d41707d12376e373d5ecc +.git +!.git/HEAD +!.git/refs + +# Common development/test artifacts +/cover/ +/doc/ +/test/ +/tmp/ +.elixir_ls + +# Mix artifacts +/_build/ +/deps/ +*.ez + +# Generated on crash by the VM +erl_crash.dump + +# Static artifacts - These should be fetched and built inside the Docker image +/assets/node_modules/ +/priv/static/assets/ +/priv/static/cache_manifest.json diff --git a/config/config.exs b/config/config.exs index 2f1c6ecb..238bf64e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -13,12 +13,12 @@ config :pinchflat, # Specifying backend data here makes mocking and local testing SUPER easy yt_dlp_executable: System.find_executable("yt-dlp"), yt_dlp_runner: Pinchflat.MediaClient.Backends.YtDlp.CommandRunner, - media_directory: "/media", + media_directory: "/downloads", metadata_directory: Path.join([System.tmp_dir!(), "pinchflat", "metadata"]) # Configures the endpoint config :pinchflat, PinchflatWeb.Endpoint, - url: [host: "localhost"], + url: [host: "localhost", port: 8945], adapter: Phoenix.Endpoint.Cowboy2Adapter, render_errors: [ formats: [html: PinchflatWeb.ErrorHTML, json: PinchflatWeb.ErrorJSON], diff --git a/config/runtime.exs b/config/runtime.exs index af11e69e..0686ac97 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -1,4 +1,5 @@ import Config +require Logger # config/runtime.exs is executed for all environments, including # during releases. It is executed after compilation and before the @@ -28,6 +29,8 @@ if config_env() == :prod do For example: /etc/pinchflat/pinchflat.db """ + config :pinchflat, yt_dlp_executable: System.find_executable("yt-dlp") + config :pinchflat, Pinchflat.Repo, database: database_path, pool_size: String.to_integer(System.get_env("POOL_SIZE") || "5") @@ -38,26 +41,29 @@ if config_env() == :prod do # to check this value into version control, so we use an environment # variable instead. secret_key_base = - System.get_env("SECRET_KEY_BASE") || - raise """ - environment variable SECRET_KEY_BASE is missing. - You can generate one by calling: mix phx.gen.secret - """ + if System.get_env("SECRET_KEY_BASE") do + System.get_env("SECRET_KEY_BASE") + else + Logger.warning(""" + Using the default SECRET_KEY_BASE in a conventional production environment + is dangerous. Please set the SECRET_KEY_BASE environment variable if you're + deploying this to an internet-facing server. If you're running this in a + private network, it's likely safe to use the default value. + """) - host = System.get_env("PHX_HOST") || "example.com" - port = String.to_integer(System.get_env("PORT") || "4000") + "ZkuQMStdmUzBv+gO3m3XZrtQW76e+AX3QIgTLajw3b/HkTLMEx+DOXr2WZsSS+n8" + end config :pinchflat, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY") config :pinchflat, PinchflatWeb.Endpoint, - url: [host: host, port: 443, scheme: "https"], http: [ # Enable IPv6 and bind on all interfaces. # Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access. # See the documentation on https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html # for details about using IPv6 vs IPv4 and loopback vs public addresses. ip: {0, 0, 0, 0, 0, 0, 0, 0}, - port: port + port: String.to_integer(System.get_env("PORT") || "4000") ], secret_key_base: secret_key_base diff --git a/Dockerfile b/dev.Dockerfile similarity index 100% rename from Dockerfile rename to dev.Dockerfile diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 19e7aae2..957fd632 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -1,7 +1,9 @@ version: '3' services: phx: - build: . + build: + context: . + dockerfile: dev.Dockerfile volumes: - '.:/app' ports: diff --git a/docker-compose.yml b/docker-compose.yml index 2f89a504..23cde706 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,14 @@ version: '3' services: phx: - build: . + build: + context: . + dockerfile: dev.Dockerfile volumes: - '.:/app' ports: - '4008:4008' command: - - ./docker-run.sh + - ./docker-run.dev.sh stdin_open: true tty: true diff --git a/docker-run.sh b/docker-run.dev.sh similarity index 100% rename from docker-run.sh rename to docker-run.dev.sh diff --git a/lib/pinchflat/application.ex b/lib/pinchflat/application.ex index 6842111a..6846c2f6 100644 --- a/lib/pinchflat/application.ex +++ b/lib/pinchflat/application.ex @@ -21,6 +21,8 @@ defmodule Pinchflat.Application do PinchflatWeb.Endpoint ] + :ok = Oban.Telemetry.attach_default_logger() + # See https://hexdocs.pm/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Pinchflat.Supervisor] diff --git a/lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex b/lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex index e88e238e..bb98b204 100644 --- a/lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex +++ b/lib/pinchflat/media_client/backends/yt_dlp/command_runner.ex @@ -30,7 +30,7 @@ defmodule Pinchflat.MediaClient.Backends.YtDlp.CommandRunner do print_to_file_opts = [{:print_to_file, output_template}, json_output_path] formatted_command_opts = [url] ++ parse_options(command_opts ++ print_to_file_opts) - Logger.debug("[yt-dlp] called with: #{Enum.join(formatted_command_opts, " ")}") + Logger.info("[yt-dlp] called with: #{Enum.join(formatted_command_opts, " ")}") case System.cmd(command, formatted_command_opts, stderr_to_stdout: true) do {_, 0} -> diff --git a/lib/pinchflat/release.ex b/lib/pinchflat/release.ex new file mode 100644 index 00000000..f98d10dc --- /dev/null +++ b/lib/pinchflat/release.ex @@ -0,0 +1,28 @@ +defmodule Pinchflat.Release do + @moduledoc """ + Used for executing DB release tasks when run in production without Mix + installed. + """ + @app :pinchflat + + def migrate do + load_app() + + for repo <- repos() do + {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true)) + end + end + + def rollback(repo, version) do + load_app() + {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version)) + end + + defp repos do + Application.fetch_env!(@app, :ecto_repos) + end + + defp load_app do + Application.load(@app) + end +end diff --git a/rel/overlays/bin/docker_start b/rel/overlays/bin/docker_start new file mode 100755 index 00000000..2a0b4f69 --- /dev/null +++ b/rel/overlays/bin/docker_start @@ -0,0 +1,5 @@ +#!/bin/sh +/app/bin/migrate + +cd -P -- "$(dirname -- "$0")" +PHX_SERVER=true exec ./pinchflat start diff --git a/rel/overlays/bin/migrate b/rel/overlays/bin/migrate new file mode 100755 index 00000000..b2fb396c --- /dev/null +++ b/rel/overlays/bin/migrate @@ -0,0 +1,3 @@ +#!/bin/sh +cd -P -- "$(dirname -- "$0")" +exec ./pinchflat eval Pinchflat.Release.migrate diff --git a/selfhosted.Dockerfile b/selfhosted.Dockerfile new file mode 100644 index 00000000..5d638964 --- /dev/null +++ b/selfhosted.Dockerfile @@ -0,0 +1,118 @@ +# Find eligible builder and runner images on Docker Hub. We use Ubuntu/Debian +# instead of Alpine to avoid DNS resolution issues in production. +# +# https://hub.docker.com/r/hexpm/elixir/tags?page=1&name=ubuntu +# https://hub.docker.com/_/ubuntu?tab=tags +# +# This file is based on these images: +# +# - https://hub.docker.com/r/hexpm/elixir/tags - for the build image +# - https://hub.docker.com/_/debian?tab=tags&page=1&name=bullseye-20231009-slim - for the release image +# - https://pkgs.org/ - resource for finding needed packages +# - Ex: hexpm/elixir:1.16.0-erlang-26.2.1-debian-bullseye-20231009-slim +# +ARG ELIXIR_VERSION=1.16.0 +ARG OTP_VERSION=26.2.1 +ARG DEBIAN_VERSION=bullseye-20231009-slim + +ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}" +ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}" + +FROM ${BUILDER_IMAGE} as builder + +# install build dependencies +RUN apt-get update -y && apt-get install -y build-essential git curl \ + && apt-get clean && rm -f /var/lib/apt/lists/*_* + +# prepare build dir +WORKDIR /app + +# Install nodejs +RUN curl -sL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh +RUN bash nodesource_setup.sh +RUN apt-get install -y nodejs +RUN npm install -g yarn + +# install hex + rebar +RUN mix local.hex --force && \ + mix local.rebar --force + +# set build ENV +ENV MIX_ENV="prod" + +# install mix dependencies +COPY mix.exs mix.lock ./ +RUN mix deps.get --only $MIX_ENV +RUN mkdir config + +# copy compile-time config files before we compile dependencies +# to ensure any relevant config change will trigger the dependencies +# to be re-compiled. +COPY config/config.exs config/${MIX_ENV}.exs config/ +RUN mix deps.compile + +COPY priv priv + +COPY lib lib + +COPY assets assets + +# compile assets +RUN yarn --cwd assets install +RUN mix assets.deploy + +# Compile the release +RUN mix compile + +# Changes to config/runtime.exs don't require recompiling the code +COPY config/runtime.exs config/ + +COPY rel rel +RUN mix release + +# start a new build stage so that the final image will only contain +# the compiled release and other runtime necessities +FROM ${RUNNER_IMAGE} + +RUN apt-get update -y +RUN apt-get install -y libstdc++6 openssl libncurses5 locales ca-certificates \ + python3 python3-pip ffmpeg +RUN apt-get clean && rm -f /var/lib/apt/lists/*_* + +# Download YT-DLP +# NOTE: If you're seeing weird issues, consider using the FFMPEG released by yt-dlp +RUN python3 -m pip install -U --pre yt-dlp + +# Set the locale +RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen + +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +WORKDIR "/app" +RUN chown nobody /app + +# Set up data volumes +RUN mkdir /config /downloads +RUN chown nobody /config /downloads +VOLUME /config +VOLUME /downloads + +# set runner ENV +ENV MIX_ENV="prod" +ENV DATABASE_PATH="/config/pinchflat.db" +ENV PORT=8945 + +# Only copy the final release from the build stage +COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/pinchflat ./ + +USER nobody + +# If using an environment that doesn't automatically reap zombie processes, it is +# advised to add an init process such as tini via `apt-get install` +# above and adding an entrypoint. See https://github.com/krallin/tini for details +# ENTRYPOINT ["/tini", "--"] + +# Start the app +CMD ["/app/bin/docker_start"] From cec5018d44fa87dd3c15fe4e4b5160b3f8426a0c Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Fri, 16 Feb 2024 11:43:57 -0800 Subject: [PATCH 3/3] Improved fetching SECRET_KEY_BASE for selfhosted instances --- config/runtime.exs | 21 ++++++++++++++------- selfhosted.Dockerfile | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 0686ac97..496ca872 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -44,14 +44,21 @@ if config_env() == :prod do if System.get_env("SECRET_KEY_BASE") do System.get_env("SECRET_KEY_BASE") else - Logger.warning(""" - Using the default SECRET_KEY_BASE in a conventional production environment - is dangerous. Please set the SECRET_KEY_BASE environment variable if you're - deploying this to an internet-facing server. If you're running this in a - private network, it's likely safe to use the default value. - """) + if System.get_env("RUN_CONTEXT") == "selfhosted" do + # Using the default SECRET_KEY_BASE in a conventional production environment + # is dangerous. Please set the SECRET_KEY_BASE environment variable if you're + # deploying this to an internet-facing server. If you're running this in a + # private network, it's likely safe to use the default value. If you want + # to be extra safe, run `mix phx.gen.secret` and set the SECRET_KEY_BASE + # environment variable to the output of that command. - "ZkuQMStdmUzBv+gO3m3XZrtQW76e+AX3QIgTLajw3b/HkTLMEx+DOXr2WZsSS+n8" + "ZkuQMStdmUzBv+gO3m3XZrtQW76e+AX3QIgTLajw3b/HkTLMEx+DOXr2WZsSS+n8" + else + raise """ + environment variable SECRET_KEY_BASE is missing. + You can generate one by calling: mix phx.gen.secret + """ + end end config :pinchflat, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY") diff --git a/selfhosted.Dockerfile b/selfhosted.Dockerfile index 5d638964..1a1508d8 100644 --- a/selfhosted.Dockerfile +++ b/selfhosted.Dockerfile @@ -103,6 +103,7 @@ VOLUME /downloads ENV MIX_ENV="prod" ENV DATABASE_PATH="/config/pinchflat.db" ENV PORT=8945 +ENV RUN_CONTEXT="selfhosted" # Only copy the final release from the build stage COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/pinchflat ./