docker-ingestion.yml

name: datahub-ingestion docker
on:
  push:
    branches:
      - master
    paths-ignore:
      - "docs/**"
      - "**.md"
  pull_request:
    branches:
      - master
    paths:
      - "metadata-ingestion/**"
      - "metadata-models/**"
      - "docker/datahub-ingestion/**"
      - "docker/datahub-ingestion-slim/**"
      - ".github/workflows/docker-ingestion.yml"
  release:
    types: [published]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
      publish: ${{ steps.publish.outputs.publish }}
      python_release_version: ${{ steps.python_release_version.outputs.release_version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Compute Tag
        id: tag
        run: |
          source .github/scripts/docker_helpers.sh
          echo "tag=$(get_tag)" >> $GITHUB_OUTPUT
      - name: Compute Python Release Version
        id: python_release_version
        run: |
          source .github/scripts/docker_helpers.sh
          echo "release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT
      - name: Check whether publishing enabled
        id: publish
        env:
          ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD }}
        run: |
          echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
          echo "publish=${{ env.ENABLE_PUBLISH != '' }}" >> $GITHUB_OUTPUT
  push_to_registries:
    name: Build and Push Docker Image to Docker Hub
    runs-on: ubuntu-latest
    needs: setup
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Build and push
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            linkedin/datahub-ingestion
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish == 'true' }}
          context: .
          file: ./docker/datahub-ingestion/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
          build-args: |
            RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
      - name: Build and Push image (slim)
        uses: ./.github/actions/docker-custom-build-and-push
        with:
          images: |
            acryldata/datahub-ingestion-slim
          tags: ${{ needs.setup.outputs.tag }}
          username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
          password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
          publish: ${{ needs.setup.outputs.publish == 'true' }}
          context: .
          file: ./docker/datahub-ingestion-slim/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
  ingestion-slim_scan:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    name: "[Monitoring] Scan datahub-ingestion-slim images for vulnerabilities"
    if: ${{ github.ref == 'refs/heads/master' }}
    runs-on: ubuntu-latest
    needs: [push_to_registries]
    steps:
      - name: Checkout # adding checkout step just to make trivy upload happy
        uses: actions/checkout@v3
      - name: Download image
        uses: ishworkh/docker-image-artifact-download@v1
        with:
          image: acryldata/datahub-ingestion-slim:latest
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@0.8.0
        env:
          TRIVY_OFFLINE_SCAN: true
        with:
          image-ref: acryldata/datahub-ingestion-slim:latest
          format: "template"
          template: "@/contrib/sarif.tpl"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          ignore-unfixed: true
          vuln-type: "os,library"
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: "trivy-results.sarif"