From 543353cdccf9f154250c22cdfa78b59a3e336e3b Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Tue, 7 May 2024 15:39:24 -0700 Subject: [PATCH 1/5] First pass at cutting down image size --- selfhosted.Dockerfile | 107 +++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 42 deletions(-) diff --git a/selfhosted.Dockerfile b/selfhosted.Dockerfile index 9a9d96b1..ce5b0176 100644 --- a/selfhosted.Dockerfile +++ b/selfhosted.Dockerfile @@ -21,30 +21,43 @@ ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}" FROM ${BUILDER_IMAGE} as builder # install build dependencies -RUN apt-get update -y && apt-get install -y build-essential git curl \ - && apt-get clean && rm -f /var/lib/apt/lists/*_* +RUN apt-get update -y && \ + # System packages + apt-get install -y \ + build-essential \ + git \ + curl && \ + # Node.js and Yarn + curl -sL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh && \ + bash nodesource_setup.sh && \ + apt-get install -y nodejs && \ + npm install -g yarn && \ + # Hex and Rebar + mix local.hex --force && \ + mix local.rebar --force && \ + # Cleanup + apt-get clean && \ + rm -f /var/lib/apt/lists/*_* + +# TODO: make this pull a different version of ffmpeg based on the architecture +# TODO: add to local dockerfile +# TODO: update docs +# TODO: update actions runner +RUN export FFMPEG_DOWNLOAD="https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz" && \ + curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg.tar.xz && \ + tar -xf /tmp/ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffmpeg" && \ + tar -xf /tmp/ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffprobe" # prepare build dir WORKDIR /app -# Install nodejs -RUN curl -sL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh -RUN bash nodesource_setup.sh -RUN apt-get install -y nodejs -RUN npm install -g yarn - -# install hex + rebar -RUN mix local.hex --force && \ - mix local.rebar --force - # set build ENV ENV MIX_ENV="prod" ENV ERL_FLAGS="+JPperf true" # install mix dependencies COPY mix.exs mix.lock ./ -RUN mix deps.get --only $MIX_ENV -RUN mkdir config +RUN mix deps.get --only $MIX_ENV && mkdir config # copy compile-time config files before we compile dependencies # to ensure any relevant config change will trigger the dependencies @@ -53,17 +66,11 @@ COPY config/config.exs config/${MIX_ENV}.exs config/ RUN mix deps.compile COPY priv priv - COPY lib lib - COPY assets assets -# compile assets -RUN yarn --cwd assets install -RUN mix assets.deploy - -# Compile the release -RUN mix compile +# Compile assets +RUN yarn --cwd assets install && mix assets.deploy && mix compile # Changes to config/runtime.exs don't require recompiling the code COPY config/runtime.exs config/ @@ -71,30 +78,46 @@ COPY config/runtime.exs config/ COPY rel rel RUN mix release -# start a new build stage so that the final image will only contain -# the compiled release and other runtime necessities +## -- Release Stage -- + FROM ${RUNNER_IMAGE} ARG PORT=8945 -RUN apt-get update -y -RUN apt-get install -y libstdc++6 openssl libncurses5 locales ca-certificates \ - ffmpeg curl git openssh-client nano python3 python3-pip jq procps -RUN apt-get clean && rm -f /var/lib/apt/lists/*_* - -# Download and update YT-DLP -RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp -RUN chmod a+rx /usr/local/bin/yt-dlp -RUN yt-dlp -U - -# Download Apprise -RUN python3 -m pip install -U apprise --break-system-packages - -# Download Mutagen for music thumbnail generation -RUN python3 -m pip install -U mutagen --break-system-packages - -# Set the locale -RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen +COPY --from=builder ./usr/local/bin/ffmpeg /usr/bin/ffmpeg +COPY --from=builder ./usr/local/bin/ffprobe /usr/bin/ffprobe + +RUN apt-get update -y && \ + # System packages + apt-get install -y \ + libstdc++6 \ + openssl \ + libncurses5 \ + locales \ + ca-certificates \ + python3-mutagen \ + curl \ + openssh-client \ + nano \ + python3 \ + pipx \ + jq \ + procps && \ + # Apprise + export PIPX_HOME=/opt/pipx && \ + export PIPX_BIN_DIR=/usr/local/bin && \ + pipx install apprise && \ + # yt-dlp + curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp && \ + chmod a+rx /usr/local/bin/yt-dlp && \ + yt-dlp -U && \ + # Set the locale + sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen && \ + # Clean up + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# More locale setup ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 From 9eded87979f86764f49974e1d0b8a068ae2abfa0 Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Tue, 7 May 2024 16:01:56 -0700 Subject: [PATCH 2/5] Made selfhosted docker work multiplatform --- dev.Dockerfile | 27 +++++++++++++++++++-------- selfhosted.Dockerfile | 36 +++++++++++------------------------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/dev.Dockerfile b/dev.Dockerfile index 460543e6..53dff554 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -5,10 +5,23 @@ ARG DEV_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEB FROM ${DEV_IMAGE} +ARG TARGETPLATFORM +RUN echo "Building for ${TARGETPLATFORM:?}" + # Install debian packages RUN apt-get update -qq -RUN apt-get install -y inotify-tools ffmpeg curl git openssh-client jq \ - python3 python3-pip python3-setuptools python3-wheel python3-dev locales procps +RUN apt-get install -y inotify-tools curl git openssh-client jq \ + python3 python3-setuptools python3-wheel python3-dev pipx \ + python3-mutagen locales procps build-essential + +# Install ffmpeg +RUN export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \ + "linux/amd64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz" ;; \ + "linux/arm64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz" ;; \ + *) echo "" ;; esac) && \ + curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg.tar.xz && \ + tar -xf /tmp/ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \ + tar -xf /tmp/ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe" # Install nodejs RUN curl -sL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh @@ -21,16 +34,14 @@ RUN mix local.hex --force RUN mix local.rebar --force # Download and update YT-DLP -# NOTE: If you're seeing weird issues, consider using the FFMPEG released by yt-dlp RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp RUN chmod a+rx /usr/local/bin/yt-dlp RUN yt-dlp -U -# Download Apprise -RUN python3 -m pip install -U apprise --break-system-packages - -# Download Mutagen for music thumbnail generation -RUN python3 -m pip install -U mutagen --break-system-packages +# Install Apprise +RUN export PIPX_HOME=/opt/pipx && \ + export PIPX_BIN_DIR=/usr/local/bin && \ + pipx install apprise # Set the locale RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen diff --git a/selfhosted.Dockerfile b/selfhosted.Dockerfile index ce5b0176..8a538ec0 100644 --- a/selfhosted.Dockerfile +++ b/selfhosted.Dockerfile @@ -1,16 +1,5 @@ # Find eligible builder and runner images on Docker Hub. We use Ubuntu/Debian # instead of Alpine to avoid DNS resolution issues in production. -# -# https://hub.docker.com/r/hexpm/elixir/tags?page=1&name=ubuntu -# https://hub.docker.com/_/ubuntu?tab=tags -# -# This file is based on these images: -# -# - https://hub.docker.com/r/hexpm/elixir/tags - for the build image -# - https://hub.docker.com/_/debian?tab=tags&page=1&name=bullseye-20231009-slim - for the release image -# - https://pkgs.org/ - resource for finding needed packages -# - Ex: hexpm/elixir:1.16.0-erlang-26.2.1-debian-bullseye-20231009-slim -# ARG ELIXIR_VERSION=1.16.2 ARG OTP_VERSION=26.2.2 ARG DEBIAN_VERSION=bookworm-20240130-slim @@ -20,6 +9,9 @@ ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}" FROM ${BUILDER_IMAGE} as builder +ARG TARGETPLATFORM +RUN echo "Building for ${TARGETPLATFORM:?}" + # install build dependencies RUN apt-get update -y && \ # System packages @@ -35,19 +27,18 @@ RUN apt-get update -y && \ # Hex and Rebar mix local.hex --force && \ mix local.rebar --force && \ + # FFmpeg + export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \ + "linux/amd64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz" ;; \ + "linux/arm64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz" ;; \ + *) echo "" ;; esac) && \ + curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg.tar.xz && \ + tar -xf /tmp/ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffmpeg" && \ + tar -xf /tmp/ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffprobe" && \ # Cleanup apt-get clean && \ rm -f /var/lib/apt/lists/*_* -# TODO: make this pull a different version of ffmpeg based on the architecture -# TODO: add to local dockerfile -# TODO: update docs -# TODO: update actions runner -RUN export FFMPEG_DOWNLOAD="https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz" && \ - curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg.tar.xz && \ - tar -xf /tmp/ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffmpeg" && \ - tar -xf /tmp/ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/local/bin/ "ffprobe" - # prepare build dir WORKDIR /app @@ -147,11 +138,6 @@ COPY --from=builder /app/_build/${MIX_ENV}/rel/pinchflat ./ # root, use --user 0:0 or something. RUN passwd -d root -# If using an environment that doesn't automatically reap zombie processes, it is -# advised to add an init process such as tini via `apt-get install` -# above and adding an entrypoint. See https://github.com/krallin/tini for details -# ENTRYPOINT ["/tini", "--"] - HEALTHCHECK --interval=120s --start-period=10s \ CMD curl --fail http://localhost:${PORT}/healthcheck || exit 1 From 1c2b867f3c43f5f8ab750c81995bdf2dd1c6d21b Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Tue, 7 May 2024 16:48:37 -0700 Subject: [PATCH 3/5] Updated compile-time value to be runtime --- .../components/custom_components/text_components.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pinchflat_web/components/custom_components/text_components.ex b/lib/pinchflat_web/components/custom_components/text_components.ex index 69ac0509..cba51ee3 100644 --- a/lib/pinchflat_web/components/custom_components/text_components.ex +++ b/lib/pinchflat_web/components/custom_components/text_components.ex @@ -84,9 +84,12 @@ defmodule PinchflatWeb.CustomComponents.TextComponents do """ attr :datetime, :any, required: true attr :format, :string, default: "%Y-%m-%d %H:%M:%S" - attr :timezone, :string, default: Application.compile_env(:pinchflat, :timezone) + attr :timezone, :string, default: nil def datetime_in_zone(assigns) do + timezone = assigns.timezone || Application.get_env(:pinchflat, :timezone) + assigns = Map.put(assigns, :timezone, timezone) + ~H""" """ From 7b8a6a9aed9c6e145c3739d0100e54830d62a9e9 Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Tue, 7 May 2024 16:59:02 -0700 Subject: [PATCH 4/5] Fixed issue with tzdata file permissions --- selfhosted.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/selfhosted.Dockerfile b/selfhosted.Dockerfile index 8a538ec0..1df3d3e7 100644 --- a/selfhosted.Dockerfile +++ b/selfhosted.Dockerfile @@ -126,6 +126,8 @@ EXPOSE ${PORT} # Only copy the final release from the build stage COPY --from=builder /app/_build/${MIX_ENV}/rel/pinchflat ./ +# Update permissions to let the app write the tzdata files +RUN chmod ugo+rw /app/lib/tzdata-*/priv/* # NEVER do this if you're running in an environment where you don't trust the user # (ie: most environments). This is only acceptable in a self-hosted environment. From a49490ceb9d7cec200d4cbe40808c4ec84e2bf51 Mon Sep 17 00:00:00 2001 From: Kieran Eglin Date: Tue, 7 May 2024 17:07:36 -0700 Subject: [PATCH 5/5] Applied better fix for tz issue --- config/prod.exs | 2 ++ selfhosted.Dockerfile | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/prod.exs b/config/prod.exs index 8ec9dec1..9c42c9de 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -7,6 +7,8 @@ import Config # before starting your production server. config :pinchflat, PinchflatWeb.Endpoint, cache_static_manifest: "priv/static/cache_manifest.json" +config :tzdata, :data_dir, "/etc/elixir_tzdata_data" + # Configures Swoosh API Client config :swoosh, api_client: Swoosh.ApiClient.Finch, finch_name: Pinchflat.Finch diff --git a/selfhosted.Dockerfile b/selfhosted.Dockerfile index 1df3d3e7..1101fc81 100644 --- a/selfhosted.Dockerfile +++ b/selfhosted.Dockerfile @@ -116,7 +116,7 @@ ENV LC_ALL en_US.UTF-8 WORKDIR "/app" # Set up data volumes -RUN mkdir /config /downloads +RUN mkdir /config /downloads /etc/elixir_tzdata_data && chmod ugo+rw /etc/elixir_tzdata_data # set runner ENV ENV MIX_ENV="prod" @@ -126,8 +126,6 @@ EXPOSE ${PORT} # Only copy the final release from the build stage COPY --from=builder /app/_build/${MIX_ENV}/rel/pinchflat ./ -# Update permissions to let the app write the tzdata files -RUN chmod ugo+rw /app/lib/tzdata-*/priv/* # NEVER do this if you're running in an environment where you don't trust the user # (ie: most environments). This is only acceptable in a self-hosted environment.