From e6ab64186ebb00a1ce49073b0e97ca3ec2aaa627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=B3=E9=88=9E?= Date: Wed, 22 May 2024 02:00:23 +0800 Subject: [PATCH] refactor: refactor docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 陳鈞 --- .dockerignore | 2 - .github/workflows/docker_publish.yml | 50 ++++++-- .github/workflows/docker_publish_dev.yml | 48 ++++++-- .hadolint.yml | 9 ++ Dockerfile | 144 ++++++++++++++++------- LivestreamRecorderService.sln | 2 +- docker-compose.dcproj | 4 + docker-compose.override.yml | 5 +- docker-compose.yml | 6 +- launchSettings.json | 11 ++ 10 files changed, 211 insertions(+), 70 deletions(-) create mode 100644 .hadolint.yml create mode 100644 launchSettings.json diff --git a/.dockerignore b/.dockerignore index 3729ff0..71a4ca7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,5 +21,3 @@ **/obj **/secrets.dev.yaml **/values.dev.yaml -LICENSE -README.md \ No newline at end of file diff --git a/.github/workflows/docker_publish.yml b/.github/workflows/docker_publish.yml index 51e9032..6f0cd4f 100644 --- a/.github/workflows/docker_publish.yml +++ b/.github/workflows/docker_publish.yml @@ -12,6 +12,11 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: +# Sets the permissions granted to the GITHUB_TOKEN for the actions in this job. +permissions: + contents: read + packages: write + # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: # This workflow contains a single job called "build" @@ -22,13 +27,13 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Docker meta id: meta_ApacheCouchDB - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ secrets.DOCKERHUB_ORGANIZATION_NAME }}/${{ github.event.repository.name }},ghcr.io/${{ github.repository }} flavor: | @@ -38,47 +43,72 @@ jobs: - name: Docker meta id: meta_AzureCosmosDB - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ secrets.DOCKERHUB_ORGANIZATION_NAME }}/${{ github.event.repository.name }},ghcr.io/${{ github.repository }} tags: | type=raw,value=AzureCosmosDB + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + # Create a Access Token and save it as as Actions secret # https://hub.docker.com/settings/security # DOCKERHUB_USERNAME # DOCKERHUB_TOKEN - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - # Create a Access Token with `read:packages` and `write:packages` scopes - # CR_PAT + # You may need to manage write and read access of GitHub Actions for repositories in the container settings. - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} - password: ${{ secrets.CR_PAT }} + password: ${{ github.token }} - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . + file: ./Dockerfile push: true + target: final tags: ${{ steps.meta_ApacheCouchDB.outputs.tags }} labels: ${{ steps.meta_ApacheCouchDB.outputs.labels }} build-args: | BUILD_CONFIGURATION=ApacheCouchDB_Release + VERSION=${{ github.ref_name }} + RELEASE=${{ github.run_number }} + platforms: linux/amd64,linux/arm64 + # Cache to regietry instead of gha to avoid the capacity limit. + cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:cache + cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:cache,mode=max + sbom: true + provenance: true - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . + file: ./Dockerfile push: true + target: final tags: ${{ steps.meta_AzureCosmosDB.outputs.tags }} labels: ${{ steps.meta_AzureCosmosDB.outputs.labels }} build-args: | BUILD_CONFIGURATION=AzureCosmosDB_Release + VERSION=${{ github.ref_name }} + RELEASE=${{ github.run_number }} + platforms: linux/amd64,linux/arm64 + # Cache to regietry instead of gha to avoid the capacity limit. + cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:cache + cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:cache,mode=max + sbom: true + provenance: true diff --git a/.github/workflows/docker_publish_dev.yml b/.github/workflows/docker_publish_dev.yml index 9b96e88..4a11ea7 100644 --- a/.github/workflows/docker_publish_dev.yml +++ b/.github/workflows/docker_publish_dev.yml @@ -10,6 +10,11 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: +# Sets the permissions granted to the GITHUB_TOKEN for the actions in this job. +permissions: + contents: read + packages: write + # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: # This workflow contains a single job called "build" @@ -20,13 +25,12 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - - name: Docker meta id: meta_ApacheCouchDB - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ secrets.DOCKERHUB_ORGANIZATION_NAME }}/${{ github.event.repository.name }},ghcr.io/${{ github.repository }} flavor: | @@ -36,18 +40,24 @@ jobs: - name: Docker meta id: meta_AzureCosmosDB - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ secrets.DOCKERHUB_ORGANIZATION_NAME }}/${{ github.event.repository.name }},ghcr.io/${{ github.repository }} tags: | type=raw,value=AzureCosmosDB_dev + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + # Create a Access Token and save it as as Actions secret # https://hub.docker.com/settings/security # DOCKERHUB_USERNAME # DOCKERHUB_TOKEN - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -55,28 +65,48 @@ jobs: # Create a Access Token with `read:packages` and `write:packages` scopes # CR_PAT - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} - password: ${{ secrets.CR_PAT }} + password: ${{ github.token }} - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . + file: ./Dockerfile push: true + target: final tags: ${{ steps.meta_ApacheCouchDB.outputs.tags }} labels: ${{ steps.meta_ApacheCouchDB.outputs.labels }} build-args: | BUILD_CONFIGURATION=ApacheCouchDB_Release + VERSION=${{ github.ref_name }} + RELEASE=${{ github.run_number }} + platforms: linux/amd64,linux/arm64 + # Cache to regietry instead of gha to avoid the capacity limit. + cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:cache + cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:cache,mode=max + sbom: true + provenance: true - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . + file: ./Dockerfile push: true + target: final tags: ${{ steps.meta_AzureCosmosDB.outputs.tags }} labels: ${{ steps.meta_AzureCosmosDB.outputs.labels }} build-args: | BUILD_CONFIGURATION=AzureCosmosDB_Release + VERSION=${{ github.ref_name }} + RELEASE=${{ github.run_number }} + platforms: linux/amd64,linux/arm64 + # Cache to regietry instead of gha to avoid the capacity limit. + cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:cache + cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:cache,mode=max + sbom: true + provenance: true diff --git a/.hadolint.yml b/.hadolint.yml new file mode 100644 index 0000000..9a0fcbc --- /dev/null +++ b/.hadolint.yml @@ -0,0 +1,9 @@ +ignored: + - DL3041 # Specify version with `dnf install -y -`. + - DL3042 # Avoid use of cache directory with pip. Use `pip install --no-cache-dir ` + - DL4006 # Set the SHELL option -o pipefail before RUN with a pipe in it + - DL3013 # Pin versions in pip. Instead of `pip install ` use `pip install ==` + - SC2015 # Note that A && B || C is not if-then-else. C may run when A is true. + - DL3006 # Always tag the version of an image explicitly + - DL3018 # Pin versions in apk add. Instead of `apk add ` use `apk add =` + - DL3008 # Pin versions in apt. Instead of `apt-get install ` use `apt-get install =` \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6ac676f..5377de0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,59 +1,121 @@ -### Debug image -### Setup the same as base image but used dotnet/runtime +# syntax=docker/dockerfile:1 +ARG UID=1654 +ARG VERSION=EDGE +ARG RELEASE=0 +ARG BUILD_CONFIGURATION=ApacheCouchDB_Release + +######################################## +# Debug stage +######################################## FROM mcr.microsoft.com/dotnet/runtime:8.0-alpine AS debug WORKDIR /app -RUN apk add --no-cache python3 && \ - apk add --no-cache --virtual build-deps musl-dev gcc g++ python3-dev py3-pip && \ - python3 -m venv /venv && \ - source /venv/bin/activate && \ - pip install --no-cache-dir yt-dlp && \ - pip uninstall -y setuptools pip && \ - apk del build-deps -ENV PATH="/venv/bin:$PATH" +# RUN mount cache for multi-arch: https://github.com/docker/buildx/issues/549#issuecomment-1788297892 +ARG TARGETARCH +ARG TARGETVARIANT + +RUN --mount=type=cache,id=apk-$TARGETARCH$TARGETVARIANT,sharing=locked,target=/var/cache/apk \ + --mount=from=ghcr.io/jim60105/static-ffmpeg-upx:7.0-1,source=/ffmpeg,target=/ffmpeg,rw \ + --mount=from=ghcr.io/jim60105/static-ffmpeg-upx:7.0-1,source=/ffprobe,target=/ffprobe,rw \ + apk update && apk add -u \ + # These branches follows the yt-dlp release + -X "https://dl-cdn.alpinelinux.org/alpine/edge/main" \ + -X "https://dl-cdn.alpinelinux.org/alpine/edge/community" \ + yt-dlp && \ + # Copy the compressed ffmpeg and ffprobe and overwrite the apk installed ones + cp /ffmpeg /usr/bin/ && \ + cp /ffprobe /usr/bin/ -COPY --link --from=mwader/static-ffmpeg:6.0 /ffmpeg /usr/local/bin/ffmpeg -COPY --link --from=mwader/static-ffmpeg:6.0 /ffprobe /usr/local/bin/ffprobe +######################################## +# Build stage +######################################## +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build -### Base image for yt-dlp -FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-alpine AS base -WORKDIR /app -RUN apk add --no-cache python3 && \ - apk add --no-cache --virtual build-deps musl-dev gcc g++ python3-dev py3-pip && \ - python3 -m venv /venv && \ - source /venv/bin/activate && \ - pip install --no-cache-dir yt-dlp && \ - pip uninstall -y setuptools pip && \ - apk del build-deps +WORKDIR /source -ENV PATH="/venv/bin:$PATH" +ARG TARGETARCH +RUN --mount=source=LivestreamRecorderService.csproj,target=LivestreamRecorderService.csproj \ + --mount=source=LivestreamRecorder.DB/LivestreamRecorder.DB.csproj,target=LivestreamRecorder.DB/LivestreamRecorder.DB.csproj \ + dotnet restore -a $TARGETARCH "LivestreamRecorderService.csproj" -COPY --link --from=mwader/static-ffmpeg:6.0 /ffmpeg /usr/local/bin/ffmpeg -COPY --link --from=mwader/static-ffmpeg:6.0 /ffprobe /usr/local/bin/ffprobe +######################################## +# Publish stage +######################################## +FROM build AS publish + +ARG BUILD_CONFIGURATION -### Build .NET -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build -ARG BUILD_CONFIGURATION=ApacheCouchDB_Release ARG TARGETARCH -WORKDIR /src +RUN --mount=source=.,target=.,rw \ + dotnet publish "LivestreamRecorderService.csproj" -a $TARGETARCH -c $BUILD_CONFIGURATION -o /app --self-contained true -COPY ["LivestreamRecorderService.csproj", "."] -COPY ["LivestreamRecorder.DB/LivestreamRecorder.DB.csproj", "LivestreamRecorder.DB/"] -RUN dotnet restore -a $TARGETARCH "LivestreamRecorderService.csproj" +######################################## +# Final stage +######################################## +FROM alpine:3 as final -FROM build AS publish -COPY . . -ARG BUILD_CONFIGURATION=ApacheCouchDB_Release +# RUN mount cache for multi-arch: https://github.com/docker/buildx/issues/549#issuecomment-1788297892 ARG TARGETARCH -RUN dotnet publish "LivestreamRecorderService.csproj" -a $TARGETARCH -c $BUILD_CONFIGURATION -o /app/publish --self-contained true +ARG TARGETVARIANT + +ARG UID +ENV APP_UID=$UID + +ENV DOTNET_RUNNING_IN_CONTAINER=true +ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true + +RUN --mount=type=cache,id=apk-$TARGETARCH$TARGETVARIANT,sharing=locked,target=/var/cache/apk \ + apk update && apk add -u \ + ca-certificates-bundle libgcc libssl3 libstdc++ zlib + +# Create directories with correct permissions +RUN install -d -m 775 -o $UID -g 0 /app && \ + install -d -m 775 -o $UID -g 0 /licenses + +# Copy licenses (OpenShift Policy) +COPY --link --chown=$UID:0 --chmod=775 LICENSE /licenses/LICENSE +ADD --link --chown=$UID:0 --chmod=775 https://raw.githubusercontent.com/yt-dlp/yt-dlp/master/LICENSE /licenses/yt-dlp.LICENSE -### Final image -FROM base AS final +RUN --mount=type=cache,id=apk-$TARGETARCH$TARGETVARIANT,sharing=locked,target=/var/cache/apk \ + --mount=from=ghcr.io/jim60105/static-ffmpeg-upx:7.0-1,source=/ffmpeg,target=/ffmpeg,rw \ + --mount=from=ghcr.io/jim60105/static-ffmpeg-upx:7.0-1,source=/ffprobe,target=/ffprobe,rw \ + --mount=from=ghcr.io/jim60105/static-ffmpeg-upx:7.0-1,source=/dumb-init,target=/dumb-init,rw \ + apk update && apk add -u \ + # These branches follows the yt-dlp release + -X "https://dl-cdn.alpinelinux.org/alpine/edge/main" \ + -X "https://dl-cdn.alpinelinux.org/alpine/edge/community" \ + yt-dlp && \ + # Copy the compressed ffmpeg and ffprobe and overwrite the apk installed ones + cp /ffmpeg /usr/bin/ && \ + cp /ffprobe /usr/bin/ && \ + cp /dumb-init /usr/bin/ + +COPY --link --chown=$UID:0 --chmod=775 --from=publish /app /app ENV PATH="/app:$PATH" -COPY --link --from=publish --chown=$APP_UID:$APP_UID /app/publish /app +WORKDIR /app + +USER $UID + +STOPSIGNAL SIGINT + +# Use dumb-init as PID 1 to handle signals properly +ENTRYPOINT [ "dumb-init", "--", "/app/LivestreamRecorderService" ] -USER $APP_UID -ENTRYPOINT ["/app/LivestreamRecorderService"] \ No newline at end of file +ARG VERSION +ARG RELEASE +LABEL name="Recorder-moe/LivestreamRecorderService" \ + # Authors for LivestreamRecorderService + vendor="Recorder-moe" \ + # Maintainer for this docker image + maintainer="jim60105" \ + # Dockerfile source repository + url="https://github.com/Recorder-moe/LivestreamRecorderService" \ + version=${VERSION} \ + # This should be a number, incremented with each change + release=${RELEASE} \ + io.k8s.display-name="LivestreamRecorderService" \ + summary="LivestreamRecorderService: The monitoring worker service for the Recorder.moe project." \ + description="Recorder.moe is an advanced live stream recording system. We utilize containerization technology to achieve horizontal scalability, enabling us to monitor and record an unlimited number of channels simultaneously. For more information about this tool, please visit the following website: https://github.com/Recorder-moe" diff --git a/LivestreamRecorderService.sln b/LivestreamRecorderService.sln index 5bb7487..9c0c041 100644 --- a/LivestreamRecorderService.sln +++ b/LivestreamRecorderService.sln @@ -41,7 +41,7 @@ Global {D9705463-12BD-4589-B8E0-53C6EE792EE8}.ApacheCouchDB_Release|Any CPU.ActiveCfg = Release|Any CPU {D9705463-12BD-4589-B8E0-53C6EE792EE8}.AzureCosmosDB_Release|Any CPU.ActiveCfg = Release|Any CPU {D9705463-12BD-4589-B8E0-53C6EE792EE8}.CosmosDB|Any CPU.ActiveCfg = Release|Any CPU - {D9705463-12BD-4589-B8E0-53C6EE792EE8}.CouchDB|Any CPU.ActiveCfg = Release|Any CPU + {D9705463-12BD-4589-B8E0-53C6EE792EE8}.CouchDB|Any CPU.ActiveCfg = CouchDB|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/docker-compose.dcproj b/docker-compose.dcproj index 0c75253..216452e 100644 --- a/docker-compose.dcproj +++ b/docker-compose.dcproj @@ -5,6 +5,10 @@ Linux d9705463-12bd-4589-b8e0-53c6ee792ee8 + + livestreamrecorderservice + + docker-compose.yml diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 5c9bde7..97bf4ed 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,7 +1,8 @@ -version: '3.4' - services: livestreamrecorderservice: + image: ghcr.io/recorder-moe/livestreamrecorderservice:debug + build: + target: debug environment: - DOTNET_ENVIRONMENT=Development volumes: diff --git a/docker-compose.yml b/docker-compose.yml index b0b7d6c..63d32ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,8 @@ -version: '3.4' - services: livestreamrecorderservice: container_name: livestreamrecorderservice image: ghcr.io/recorder-moe/livestreamrecorderservice:latest - #build: - # context: . - # dockerfile: Dockerfile + user: "1654:0" env_file: - .env volumes: diff --git a/launchSettings.json b/launchSettings.json new file mode 100644 index 0000000..63f8eed --- /dev/null +++ b/launchSettings.json @@ -0,0 +1,11 @@ +{ + "profiles": { + "Docker Compose": { + "commandName": "DockerCompose", + "commandVersion": "1.0", + "serviceActions": { + "livestreamrecorderservice": "StartDebugging" + } + } + } +} \ No newline at end of file