From 994cb3ad7728b991318be2dae799af1683cc2c43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=B3=E9=88=9E?= Date: Sat, 2 Dec 2023 20:08:30 +0800 Subject: [PATCH 1/3] feat: Add Dockerfile.ubi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add a new Dockerfile called Dockerfile.ubi - Build a base image from registry.access.redhat.com/ubi9/ubi-minimal - Set the environment variable TZ to Asia/Taipei - Install python3.11 and tzdata using microdnf - Create a symbolic link to python3.11 - Create a virtual environment at /venv and update the PATH environment variable - Build yt-dlp in a separate build stage - Install python3.11-pip and yt-dlp using microdnf and pip3.11 respectively - Remove python3.11-pip and clean up - Build .NET using registry.access.redhat.com/ubi8/dotnet-80 - Set the user to 0 and set the working directory to /src - Copy the project file and restore dependencies - Copy the remaining files and publish the project - Build the final image from the base image - Set environment variables to disable file locking and run the app in globalization-invariant mode - Set the working directory to /app - Copy the published files and the virtual environment from the build stages - Set the user to 0 and configure the entry point as ./YoutubeLiveChatToDiscord $@ Signed-off-by: 陳鈞 --- Dockerfile.ubi | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 Dockerfile.ubi diff --git a/Dockerfile.ubi b/Dockerfile.ubi new file mode 100644 index 0000000..396104b --- /dev/null +++ b/Dockerfile.ubi @@ -0,0 +1,50 @@ +### Base image +FROM registry.access.redhat.com/ubi9/ubi-minimal AS base + +ENV TZ=Asia/Taipei +RUN microdnf -y install python3.11 tzdata && \ + microdnf clean all && \ + ln -s /usr/bin/python3.11 /usr/bin/python3 + +RUN python3 -m venv /venv +ENV PATH="/venv/bin:$PATH" + +### Build yt-dlp +FROM base AS build_ytdlp + +RUN microdnf -y install python3.11-pip && \ + pip3.11 install yt-dlp && \ + microdnf -y remove python3.11-pip && \ + microdnf clean all + +### Build .NET +FROM registry.access.redhat.com/ubi8/dotnet-80 AS build + +USER 0 +WORKDIR /src + +COPY YoutubeLiveChatToDiscord.csproj . +RUN dotnet restore "./YoutubeLiveChatToDiscord.csproj" +COPY . . + +RUN dotnet publish "YoutubeLiveChatToDiscord.csproj" -c Release -o /src/publish -p:PublishTrimmed=true --self-contained true + +### Final image +FROM base AS final + +# Disable file locking on Unix +# https://github.com/dotnet/runtime/issues/34126#issuecomment-1104981659 +ENV DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true + +# Determines whether a .NET Core app runs in globalization-invariant mode without access to culture-specific data and behavior. +# https://aka.ms/dotnet-missing-libicu +ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true + +WORKDIR /app + +COPY --link --chown=1001:1001 --from=build /src/publish . +COPY --link --from=build_ytdlp /venv /venv + +USER 0 + +ENTRYPOINT ./YoutubeLiveChatToDiscord $@ \ No newline at end of file From 419ad1753b1030e4b95d144a08f5a51e9f0f9928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=B3=E9=88=9E?= Date: Sun, 3 Dec 2023 02:28:59 +0800 Subject: [PATCH 2/3] chore: Self-contained and trim publish for smaller size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Modify the Dockerfile to install python3 in image - Increase the name of the base image for Docker from ubi8 to ubi9 - Modify the Dockerfile to remove the installation of g++ and npm - Modify the Dockerfile to remove unnecessary commands related to venv - Modify the Dockerfile.ubi to install python3.11 instead of python3.8 - Modify the Dockerfile.ubi to remove unnecessary commands related to venv - Modify the Dockerfile.ubi to remove unnecessary commands related to python3.11-pip - Modify the Dockerfile.ubi to install dotnet-sdk-8.0 for debug image - Modify the Dockerfile.ubi to set the base image as python and update dependencies - Modify the Helper.cs file to set the path of yt-dlp to /venv/bin/yt-dlp - Modify the YoutubeLiveChatToDiscord.csproj file to enable publish trimming and partial trim mode Signed-off-by: 陳鈞 --- Dockerfile | 24 ++++++---- Dockerfile.ubi | 85 ++++++++++++++++++++++----------- Helper.cs | 2 +- YoutubeLiveChatToDiscord.csproj | 5 +- 4 files changed, 79 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2f29950..5210940 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,31 +2,39 @@ FROM mcr.microsoft.com/dotnet/runtime:8.0-alpine AS base WORKDIR /app -RUN apk add --no-cache --virtual build-deps musl-dev gcc g++ python3-dev &&\ - apk add --no-cache py3-pip tzdata &&\ - pip install yt-dlp &&\ +RUN apk add --no-cache tzdata python3 && \ + apk add --no-cache --virtual build-deps musl-dev gcc g++ python3-dev py3-pip && \ + python3 -m venv /venv && \ + source /venv/bin/activate && \ + pip install yt-dlp && \ + pip uninstall -y setuptools pip && \ apk del build-deps + +ENV PATH="/venv/bin:$PATH" ENV TZ=Asia/Taipei # Disable file locking on Unix # https://github.com/dotnet/runtime/issues/34126#issuecomment-1104981659 ENV DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true +FROM base AS debug + FROM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build ARG BUILD_CONFIGURATION=Release WORKDIR /src COPY ["YoutubeLiveChatToDiscord.csproj", "."] +ARG TARGETPLATFORM RUN dotnet restore "YoutubeLiveChatToDiscord.csproj" -COPY . . -RUN dotnet build "YoutubeLiveChatToDiscord.csproj" -c $BUILD_CONFIGURATION -o /app/build FROM build AS publish +COPY . . ARG BUILD_CONFIGURATION=Release -RUN dotnet publish "YoutubeLiveChatToDiscord.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false +ARG TARGETPLATFORM +RUN dotnet publish "YoutubeLiveChatToDiscord.csproj" --no-self-contained -p:PublishTrimmed=false -c $BUILD_CONFIGURATION -o /app/publish FROM base AS final WORKDIR /app COPY --from=publish /app/publish . -RUN mkdir -p /app && chown -R app:app /app -USER app +RUN chown -R 1001:1001 /app +USER 1001 ENTRYPOINT ["dotnet", "YoutubeLiveChatToDiscord.dll"] \ No newline at end of file diff --git a/Dockerfile.ubi b/Dockerfile.ubi index 396104b..9b32ec0 100644 --- a/Dockerfile.ubi +++ b/Dockerfile.ubi @@ -1,50 +1,81 @@ +### Python +FROM registry.access.redhat.com/ubi9/ubi-minimal AS python + +ENV PYTHON_VERSION=3.11 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONIOENCODING=UTF-8 +ARG PIP_DISABLE_PIP_VERSION_CHECK=1 +ARG PIP_NO_CACHE_DIR=1 + +RUN microdnf -y install python3.11 python3.11-pip && \ + microdnf -y clean all + +RUN python3.11 -m venv /venv && \ + source /venv/bin/activate &&\ + pip3.11 install yt-dlp && \ + pip3.11 uninstall -y setuptools pip && \ + microdnf -y remove python3.11-pip && \ + microdnf -y clean all && \ + rm -rf /var/cache/yum/* + +ENV PATH="/venv/bin:$PATH" + ### Base image -FROM registry.access.redhat.com/ubi9/ubi-minimal AS base +FROM python AS base + +WORKDIR /app + +RUN microdnf -y install libicu tzdata && \ + microdnf -y clean all && \ + rm -rf /var/cache/yum/* ENV TZ=Asia/Taipei -RUN microdnf -y install python3.11 tzdata && \ - microdnf clean all && \ - ln -s /usr/bin/python3.11 /usr/bin/python3 -RUN python3 -m venv /venv -ENV PATH="/venv/bin:$PATH" +# Disable file locking on Unix +# https://github.com/dotnet/runtime/issues/34126#issuecomment-1104981659 +ENV DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true -### Build yt-dlp -FROM base AS build_ytdlp -RUN microdnf -y install python3.11-pip && \ - pip3.11 install yt-dlp && \ - microdnf -y remove python3.11-pip && \ - microdnf clean all +### Debug image +FROM base AS debug + +# Install .NET 8 SDK +RUN microdnf -y install dotnet-sdk-8.0 + ### Build .NET FROM registry.access.redhat.com/ubi8/dotnet-80 AS build USER 0 +ARG BUILD_CONFIGURATION=Release WORKDIR /src COPY YoutubeLiveChatToDiscord.csproj . -RUN dotnet restore "./YoutubeLiveChatToDiscord.csproj" +ARG TARGETPLATFORM +RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \ + dotnet restore "YoutubeLiveChatToDiscord.csproj" -r linux-x64; \ + elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ + dotnet restore "YoutubeLiveChatToDiscord.csproj" -r linux-arm64; \ + fi + +FROM build AS publish COPY . . +ARG BUILD_CONFIGURATION=Release +ARG TARGETPLATFORM +RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \ + dotnet publish "YoutubeLiveChatToDiscord.csproj" -c $BUILD_CONFIGURATION -o /app/publish -r linux-x64; \ + elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ + dotnet publish "YoutubeLiveChatToDiscord.csproj" -c $BUILD_CONFIGURATION -o /app/publish -r linux-arm64; \ + fi -RUN dotnet publish "YoutubeLiveChatToDiscord.csproj" -c Release -o /src/publish -p:PublishTrimmed=true --self-contained true ### Final image FROM base AS final -# Disable file locking on Unix -# https://github.com/dotnet/runtime/issues/34126#issuecomment-1104981659 -ENV DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true - -# Determines whether a .NET Core app runs in globalization-invariant mode without access to culture-specific data and behavior. -# https://aka.ms/dotnet-missing-libicu -ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true +ENV PATH="/app:$PATH" -WORKDIR /app - -COPY --link --chown=1001:1001 --from=build /src/publish . -COPY --link --from=build_ytdlp /venv /venv +COPY --link --chown=1001:1001 --from=publish /app/publish . -USER 0 +USER 1001 -ENTRYPOINT ./YoutubeLiveChatToDiscord $@ \ No newline at end of file +ENTRYPOINT ["YoutubeLiveChatToDiscord"] \ No newline at end of file diff --git a/Helper.cs b/Helper.cs index a8aaf4e..3b277bc 100644 --- a/Helper.cs +++ b/Helper.cs @@ -31,7 +31,7 @@ public static string WhereIsYt_dlp() from e in extensions let path = Path.Combine(p.Trim(), file + e.ToLower()) where File.Exists(path) - select path)?.FirstOrDefault() ?? "/usr/bin/yt-dlp"; + select path)?.FirstOrDefault() ?? "/venv/bin/yt-dlp"; _logger.LogDebug("Found yt-dlp at {path}", YtdlPath); return YtdlPath; } diff --git a/YoutubeLiveChatToDiscord.csproj b/YoutubeLiveChatToDiscord.csproj index 633538f..6bd4f20 100644 --- a/YoutubeLiveChatToDiscord.csproj +++ b/YoutubeLiveChatToDiscord.csproj @@ -1,4 +1,4 @@ - + net8.0 enable @@ -7,6 +7,9 @@ Linux . false + debug + true + partial From aab9e1f2df6d7dba34c9af8fb5a7ec55424035b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=B3=E9=88=9E?= Date: Sun, 3 Dec 2023 06:38:49 +0800 Subject: [PATCH 3/3] ci: Update actions, add multi-arch support & scan workflow --- .github/workflows/docker_publish.yml | 20 ++-- .github/workflows/scan.yml | 70 +++++++++++++ .github/workflows/scan/html.tpl | 149 +++++++++++++++++++++++++++ 3 files changed, 233 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/scan.yml create mode 100644 .github/workflows/scan/html.tpl diff --git a/.github/workflows/docker_publish.yml b/.github/workflows/docker_publish.yml index ceefa6f..4e5e42e 100644 --- a/.github/workflows/docker_publish.yml +++ b/.github/workflows/docker_publish.yml @@ -4,6 +4,10 @@ name: docker_publish on: # Triggers the workflow on push or pull request events but only for the master branch push: + branches: + - "master" + tags: + - "*" # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -18,13 +22,13 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true - name: Docker meta id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v5 with: images: ${{ secrets.DOCKERHUB_USERNAME }}/${{ github.event.repository.name }},ghcr.io/${{ github.repository }},quay.io/${{ github.repository }} # set latest tag for default branch @@ -33,12 +37,15 @@ jobs: type=ref,event=tag type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + # Create a Access Token and save it as as Actions secret # https://hub.docker.com/settings/security # DOCKERHUB_USERNAME # DOCKERHUB_TOKEN - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -46,23 +53,24 @@ jobs: # Create a Access Token with `read:packages` and `write:packages` scopes # CR_PAT - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.CR_PAT }} - name: Login to Quay Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: . push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64, linux/arm64 diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml new file mode 100644 index 0000000..c200e3a --- /dev/null +++ b/.github/workflows/scan.yml @@ -0,0 +1,70 @@ +name: scan + +on: + workflow_run: + workflows: [docker_publish] + types: [completed] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + scan-python: + name: Scan Microsoft official base image + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/workflows/scan/html.tpl + sparse-checkout-cone-mode: false + + - name: Run Trivy vulnerability scanner for Microsoft official image + uses: aquasecurity/trivy-action@0.14.0 + with: + image-ref: "ghcr.io/jim60105/youtubelivechattodiscord:latest" + vuln-type: "os,library" + scanners: vuln + severity: "CRITICAL,HIGH" + format: "template" + template: "@.github/workflows/scan/html.tpl" + output: "trivy-results-microsoft.html" + + - name: Upload Artifact + uses: actions/upload-artifact@v3 + with: + name: trivy-results + path: trivy-results-microsoft.html + retention-days: 90 + + scan-ubi: + name: Scan Red Hat UBI base image + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/workflows/scan/html.tpl + sparse-checkout-cone-mode: false + + - name: Run Trivy vulnerability scanner for UBI image + uses: aquasecurity/trivy-action@0.14.0 + with: + image-ref: "ghcr.io/jim60105/youtubelivechattodiscord:ubi" + vuln-type: "os,library" + scanners: vuln + severity: "CRITICAL,HIGH" + format: "template" + template: "@.github/workflows/scan/html.tpl" + output: "trivy-results-ubi.html" + + - name: Upload Artifact + uses: actions/upload-artifact@v3 + with: + name: trivy-results + path: trivy-results-ubi.html + retention-days: 90 diff --git a/.github/workflows/scan/html.tpl b/.github/workflows/scan/html.tpl new file mode 100644 index 0000000..1b7d044 --- /dev/null +++ b/.github/workflows/scan/html.tpl @@ -0,0 +1,149 @@ + + + + + +{{- if . }} + + {{- escapeXML ( index . 0 ).Target }} - Trivy Report - {{ now }} + + + +

{{- escapeXML ( index . 0 ).Target }} - Trivy Report - {{ now }}

+ + {{- range . }} + + {{- if (eq (len .Vulnerabilities) 0) }} + + {{- else }} + + + + + + + + + {{- range .Vulnerabilities }} + + + + + + + + + {{- end }} + {{- end }} + {{- if (eq (len .Misconfigurations ) 0) }} + + {{- else }} + + + + + + + + {{- range .Misconfigurations }} + + + + + + + + {{- end }} + {{- end }} + {{- end }} +
{{ .Type | toString | escapeXML }}
No Vulnerabilities found
PackageVulnerability IDSeverityInstalled VersionFixed VersionLinks
{{ escapeXML .PkgName }}{{ escapeXML .VulnerabilityID }}{{ escapeXML .Vulnerability.Severity }}{{ escapeXML .InstalledVersion }}{{ escapeXML .FixedVersion }} + {{- range .Vulnerability.References }} + {{ escapeXML . }} + {{- end }} +
No Misconfigurations found
TypeMisconf IDCheckSeverityMessage
{{ escapeXML .Type }}{{ escapeXML .ID }}{{ escapeXML .Title }}{{ escapeXML .Severity }} + {{ escapeXML .Message }} +
+ {{ escapeXML .PrimaryURL }} +
+
+{{- else }} + + +

Trivy Returned Empty Report

+{{- end }} + +