feat: add ingestion overview pages (#9210) #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Docker Build, Scan, Test | |
| on: | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - "docs/**" | |
| - "**.md" | |
| pull_request: | |
| branches: | |
| - "**" | |
| paths-ignore: | |
| - "docs/**" | |
| - "**.md" | |
| release: | |
| types: [published] | |
| concurrency: | |
| # Using `github.run_id` (unique val) instead of `github.ref` here | |
| # because we don't want to cancel this workflow on master only for PRs | |
| # as that makes reproducing issues easier | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} | |
| cancel-in-progress: true | |
| env: | |
| DATAHUB_GMS_IMAGE: "linkedin/datahub-gms" | |
| DATAHUB_FRONTEND_IMAGE: "linkedin/datahub-frontend-react" | |
| DATAHUB_MAE_CONSUMER_IMAGE: "linkedin/datahub-mae-consumer" | |
| DATAHUB_MCE_CONSUMER_IMAGE: "linkedin/datahub-mce-consumer" | |
| DATAHUB_KAFKA_SETUP_IMAGE: "linkedin/datahub-kafka-setup" | |
| DATAHUB_ELASTIC_SETUP_IMAGE: "linkedin/datahub-elasticsearch-setup" | |
| DATAHUB_MYSQL_SETUP_IMAGE: "acryldata/datahub-mysql-setup" | |
| DATAHUB_UPGRADE_IMAGE: "acryldata/datahub-upgrade" | |
| DATAHUB_INGESTION_BASE_IMAGE: "acryldata/datahub-ingestion-base" | |
| DATAHUB_INGESTION_IMAGE: "acryldata/datahub-ingestion" | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tag: ${{ steps.tag.outputs.tag }} | |
| slim_tag: ${{ steps.tag.outputs.slim_tag }} | |
| full_tag: ${{ steps.tag.outputs.full_tag }} | |
| unique_tag: ${{ steps.tag.outputs.unique_tag }} | |
| unique_slim_tag: ${{ steps.tag.outputs.unique_slim_tag }} | |
| unique_full_tag: ${{ steps.tag.outputs.unique_full_tag }} | |
| publish: ${{ steps.publish.outputs.publish }} | |
| python_release_version: ${{ steps.tag.outputs.python_release_version }} | |
| short_sha: ${{ steps.tag.outputs.short_sha }} | |
| branch_name: ${{ steps.tag.outputs.branch_name }} | |
| repository_name: ${{ steps.tag.outputs.repository_name }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Compute Tag | |
| id: tag | |
| run: | | |
| source .github/scripts/docker_helpers.sh | |
| echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT | |
| echo "tag=$(get_tag)" >> $GITHUB_OUTPUT | |
| echo "slim_tag=$(get_tag_slim)" >> $GITHUB_OUTPUT | |
| echo "full_tag=$(get_tag_full)" >> $GITHUB_OUTPUT | |
| echo "unique_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT | |
| echo "unique_slim_tag=$(get_unique_tag_slim)" >> $GITHUB_OUTPUT | |
| echo "unique_full_tag=$(get_unique_tag_full)" >> $GITHUB_OUTPUT | |
| echo "python_release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT | |
| echo "branch_name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT | |
| echo "repository_name=${GITHUB_REPOSITORY#*/}" >> $GITHUB_OUTPUT | |
| - name: Check whether publishing enabled | |
| id: publish | |
| env: | |
| ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD != '' && secrets.ACRYL_DOCKER_PASSWORD != '' }} | |
| run: | | |
| echo "Enable publish: ${{ env.ENABLE_PUBLISH }}" | |
| echo "publish=${{ env.ENABLE_PUBLISH }}" >> $GITHUB_OUTPUT | |
| gms_build: | |
| name: Build and Push DataHub GMS Docker Image | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - name: Pre-build artifacts for docker image | |
| run: | | |
| ./gradlew :metadata-service:war:build -x test --parallel | |
| mv ./metadata-service/war/build/libs/war.war . | |
| - name: Build and push | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| images: | | |
| ${{ env.DATAHUB_GMS_IMAGE }} | |
| tags: ${{ needs.setup.outputs.tag }} | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/datahub-gms/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| gms_scan: | |
| permissions: | |
| contents: read # for actions/checkout to fetch code | |
| security-events: write # for github/codeql-action/upload-sarif to upload SARIF results | |
| actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status | |
| name: "[Monitoring] Scan GMS images for vulnerabilities" | |
| runs-on: ubuntu-latest | |
| needs: [setup, gms_build] | |
| steps: | |
| - name: Checkout # adding checkout step just to make trivy upload happy | |
| uses: actions/checkout@v3 | |
| - name: Download image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Run Trivy vulnerability scanner | |
| uses: aquasecurity/[email protected] | |
| env: | |
| TRIVY_OFFLINE_SCAN: true | |
| with: | |
| image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| format: "template" | |
| template: "@/contrib/sarif.tpl" | |
| output: "trivy-results.sarif" | |
| severity: "CRITICAL,HIGH" | |
| ignore-unfixed: true | |
| vuln-type: "os,library" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: "trivy-results.sarif" | |
| mae_consumer_build: | |
| name: Build and Push DataHub MAE Consumer Docker Image | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - name: Pre-build artifacts for docker image | |
| run: | | |
| ./gradlew :metadata-jobs:mae-consumer-job:build -x test --parallel | |
| mv ./metadata-jobs/mae-consumer-job/build/libs/mae-consumer-job.jar . | |
| - name: Build and push | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| images: | | |
| ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }} | |
| tags: ${{ needs.setup.outputs.tag }} | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/datahub-mae-consumer/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| mae_consumer_scan: | |
| name: "[Monitoring] Scan MAE consumer images for vulnerabilities" | |
| runs-on: ubuntu-latest | |
| needs: [setup, mae_consumer_build] | |
| permissions: | |
| contents: read # for actions/checkout to fetch code | |
| security-events: write # for github/codeql-action/upload-sarif to upload SARIF results | |
| actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status | |
| steps: | |
| - name: Checkout # adding checkout step just to make trivy upload happy | |
| uses: actions/checkout@v3 | |
| - name: Download image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Run Trivy vulnerability scanner | |
| uses: aquasecurity/[email protected] | |
| env: | |
| TRIVY_OFFLINE_SCAN: true | |
| with: | |
| image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| format: "template" | |
| template: "@/contrib/sarif.tpl" | |
| output: "trivy-results.sarif" | |
| severity: "CRITICAL,HIGH" | |
| ignore-unfixed: true | |
| vuln-type: "os,library" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: "trivy-results.sarif" | |
| mce_consumer_build: | |
| name: Build and Push DataHub MCE Consumer Docker Image | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - name: Pre-build artifacts for docker image | |
| run: | | |
| ./gradlew :metadata-jobs:mce-consumer-job:build -x test --parallel | |
| mv ./metadata-jobs/mce-consumer-job/build/libs/mce-consumer-job.jar . | |
| - name: Build and push | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| images: | | |
| ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }} | |
| tags: ${{ needs.setup.outputs.tag }} | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/datahub-mce-consumer/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| mce_consumer_scan: | |
| name: "[Monitoring] Scan MCE consumer images for vulnerabilities" | |
| runs-on: ubuntu-latest | |
| needs: [setup, mce_consumer_build] | |
| permissions: | |
| contents: read # for actions/checkout to fetch code | |
| security-events: write # for github/codeql-action/upload-sarif to upload SARIF results | |
| actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status | |
| steps: | |
| - name: Checkout # adding checkout step just to make trivy upload happy | |
| uses: actions/checkout@v3 | |
| - name: Download image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Run Trivy vulnerability scanner | |
| uses: aquasecurity/[email protected] | |
| env: | |
| TRIVY_OFFLINE_SCAN: true | |
| with: | |
| image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| format: "template" | |
| template: "@/contrib/sarif.tpl" | |
| output: "trivy-results.sarif" | |
| severity: "CRITICAL,HIGH" | |
| ignore-unfixed: true | |
| vuln-type: "os,library" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: "trivy-results.sarif" | |
| datahub_upgrade_build: | |
| name: Build and Push DataHub Upgrade Docker Image | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - name: Pre-build artifacts for docker image | |
| run: | | |
| ./gradlew :datahub-upgrade:build -x test --parallel | |
| mv ./datahub-upgrade/build/libs/datahub-upgrade.jar . | |
| - name: Build and push | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| images: | | |
| ${{ env.DATAHUB_UPGRADE_IMAGE }} | |
| tags: ${{ needs.setup.outputs.tag }} | |
| username: ${{ secrets.ACRYL_DOCKER_USERNAME }} | |
| password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/datahub-upgrade/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| datahub_upgrade_scan: | |
| name: "[Monitoring] Scan DataHub Upgrade images for vulnerabilities" | |
| runs-on: ubuntu-latest | |
| needs: [setup, datahub_upgrade_build] | |
| permissions: | |
| contents: read # for actions/checkout to fetch code | |
| security-events: write # for github/codeql-action/upload-sarif to upload SARIF results | |
| actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status | |
| steps: | |
| - name: Checkout # adding checkout step just to make trivy upload happy | |
| uses: actions/checkout@v3 | |
| - name: Download image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Run Trivy vulnerability scanner | |
| uses: aquasecurity/[email protected] | |
| env: | |
| TRIVY_OFFLINE_SCAN: true | |
| with: | |
| image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| format: "template" | |
| template: "@/contrib/sarif.tpl" | |
| output: "trivy-results.sarif" | |
| severity: "CRITICAL,HIGH" | |
| ignore-unfixed: true | |
| vuln-type: "os,library" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: "trivy-results.sarif" | |
| frontend_build: | |
| name: Build and Push DataHub Frontend Docker Image | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - name: Pre-build artifacts for docker image | |
| run: | | |
| ./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel | |
| mv ./datahub-frontend/build/distributions/datahub-frontend-*.zip datahub-frontend.zip | |
| env: | |
| NODE_OPTIONS: "--max-old-space-size=3072" | |
| - name: Build and push | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| images: | | |
| ${{ env.DATAHUB_FRONTEND_IMAGE }} | |
| tags: ${{ needs.setup.outputs.tag }} | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/datahub-frontend/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| frontend_scan: | |
| name: "[Monitoring] Scan Frontend images for vulnerabilities" | |
| runs-on: ubuntu-latest | |
| needs: [setup, frontend_build] | |
| permissions: | |
| contents: read # for actions/checkout to fetch code | |
| security-events: write # for github/codeql-action/upload-sarif to upload SARIF results | |
| actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status | |
| steps: | |
| - name: Checkout # adding checkout step just to make trivy upload happy | |
| uses: actions/checkout@v3 | |
| - name: Download image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Run Trivy vulnerability scanner | |
| uses: aquasecurity/[email protected] | |
| env: | |
| TRIVY_OFFLINE_SCAN: true | |
| with: | |
| image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| format: "template" | |
| template: "@/contrib/sarif.tpl" | |
| output: "trivy-results.sarif" | |
| severity: "CRITICAL,HIGH" | |
| ignore-unfixed: true | |
| vuln-type: "os,library" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: "trivy-results.sarif" | |
| kafka_setup_build: | |
| name: Build and Push DataHub Kafka Setup Docker Image | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - name: Build and push | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| images: | | |
| ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }} | |
| tags: ${{ needs.setup.outputs.tag }} | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/kafka-setup/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| mysql_setup_build: | |
| name: Build and Push DataHub MySQL Setup Docker Image | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - name: Build and push | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| images: | | |
| ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }} | |
| tags: ${{ needs.setup.outputs.tag }} | |
| username: ${{ secrets.ACRYL_DOCKER_USERNAME }} | |
| password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/mysql-setup/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| elasticsearch_setup_build: | |
| name: Build and Push DataHub Elasticsearch Setup Docker Image | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - name: Build and push | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| images: | | |
| ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }} | |
| tags: ${{ needs.setup.outputs.tag }} | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/elasticsearch-setup/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| datahub_ingestion_base_build: | |
| name: Build and Push DataHub Ingestion (Base) Docker Image | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tag: ${{ steps.tag.outputs.tag }} | |
| needs: setup | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| datahub-ingestion-base: | |
| - 'docker/datahub-ingestion-base/**' | |
| - name: Build and push Base Image | |
| if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' }} | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| target: base | |
| images: | | |
| ${{ env.DATAHUB_INGESTION_BASE_IMAGE }} | |
| tags: ${{ needs.setup.outputs.tag }} | |
| username: ${{ secrets.ACRYL_DOCKER_USERNAME }} | |
| password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/datahub-ingestion-base/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| - name: Compute DataHub Ingestion (Base) Tag | |
| id: tag | |
| run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }}" >> $GITHUB_OUTPUT | |
| datahub_ingestion_base_slim_build: | |
| name: Build and Push DataHub Ingestion (Base-Slim) Docker Image | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tag: ${{ steps.tag.outputs.tag }} | |
| needs: [setup, datahub_ingestion_base_build] | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| datahub-ingestion-base: | |
| - 'docker/datahub-ingestion-base/**' | |
| - name: Download Base Image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }} | |
| - name: Build and push Base-Slim Image | |
| if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' }} | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| target: slim-install | |
| images: | | |
| ${{ env.DATAHUB_INGESTION_BASE_IMAGE }} | |
| tags: ${{ needs.setup.outputs.slim_tag }} | |
| username: ${{ secrets.ACRYL_DOCKER_USERNAME }} | |
| password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} | |
| build-args: | | |
| APP_ENV=slim | |
| BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/datahub-ingestion-base/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| - name: Compute DataHub Ingestion (Base-Slim) Tag | |
| id: tag | |
| run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT | |
| datahub_ingestion_base_full_build: | |
| name: Build and Push DataHub Ingestion (Base-Full) Docker Image | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tag: ${{ steps.tag.outputs.tag }} | |
| needs: [setup, datahub_ingestion_base_build] | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| datahub-ingestion-base: | |
| - 'docker/datahub-ingestion-base/**' | |
| - name: Download Base Image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }} | |
| - name: Build and push (Base-Full) Image | |
| if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' }} | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| target: full-install | |
| images: | | |
| ${{ env.DATAHUB_INGESTION_BASE_IMAGE }} | |
| tags: ${{ needs.setup.outputs.full_tag }} | |
| username: ${{ secrets.ACRYL_DOCKER_USERNAME }} | |
| password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} | |
| build-args: | | |
| APP_ENV=full | |
| BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/datahub-ingestion-base/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| - name: Compute DataHub Ingestion (Base-Full) Tag | |
| id: tag | |
| run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_full_tag || 'head' }}" >> $GITHUB_OUTPUT | |
| datahub_ingestion_slim_build: | |
| name: Build and Push DataHub Ingestion Docker Images | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tag: ${{ steps.tag.outputs.tag }} | |
| needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }} | |
| needs: [setup, datahub_ingestion_base_slim_build] | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| datahub-ingestion-base: | |
| - 'docker/datahub-ingestion-base/**' | |
| datahub-ingestion: | |
| - 'docker/datahub-ingestion/**' | |
| - name: Build codegen | |
| if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} | |
| run: ./gradlew :metadata-ingestion:codegen | |
| - name: Download Base Image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }} | |
| - name: Build and push Slim Image | |
| if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| target: final | |
| images: | | |
| ${{ env.DATAHUB_INGESTION_IMAGE }} | |
| build-args: | | |
| BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }} | |
| DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }} | |
| RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }} | |
| APP_ENV=slim | |
| tags: ${{ needs.setup.outputs.slim_tag }} | |
| username: ${{ secrets.ACRYL_DOCKER_USERNAME }} | |
| password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/datahub-ingestion/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| - name: Compute Tag | |
| id: tag | |
| run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT | |
| datahub_ingestion_slim_scan: | |
| permissions: | |
| contents: read # for actions/checkout to fetch code | |
| security-events: write # for github/codeql-action/upload-sarif to upload SARIF results | |
| actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status | |
| name: "[Monitoring] Scan Datahub Ingestion Slim images for vulnerabilities" | |
| runs-on: ubuntu-latest | |
| needs: [setup, datahub_ingestion_slim_build] | |
| steps: | |
| - name: Checkout # adding checkout step just to make trivy upload happy | |
| uses: actions/checkout@v3 | |
| - name: Download image Slim Image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }} | |
| - name: Run Trivy vulnerability scanner Slim Image | |
| uses: aquasecurity/[email protected] | |
| env: | |
| TRIVY_OFFLINE_SCAN: true | |
| with: | |
| image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }} | |
| format: "template" | |
| template: "@/contrib/sarif.tpl" | |
| output: "trivy-results.sarif" | |
| severity: "CRITICAL,HIGH" | |
| ignore-unfixed: true | |
| vuln-type: "os,library" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: "trivy-results.sarif" | |
| datahub_ingestion_full_build: | |
| name: Build and Push DataHub Ingestion (Full) Docker Images | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tag: ${{ steps.tag.outputs.tag }} | |
| needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }} | |
| needs: [setup, datahub_ingestion_base_full_build] | |
| steps: | |
| - name: Check out the repo | |
| uses: hsheth2/sane-checkout-action@v1 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| datahub-ingestion-base: | |
| - 'docker/datahub-ingestion-base/**' | |
| datahub-ingestion: | |
| - 'docker/datahub-ingestion/**' | |
| - name: Build codegen | |
| if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} | |
| run: ./gradlew :metadata-ingestion:codegen | |
| - name: Download Base Image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }} | |
| - name: Build and push Full Image | |
| if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }} | |
| uses: ./.github/actions/docker-custom-build-and-push | |
| with: | |
| target: final | |
| images: | | |
| ${{ env.DATAHUB_INGESTION_IMAGE }} | |
| build-args: | | |
| BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }} | |
| DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_tag || 'head' }} | |
| RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }} | |
| tags: ${{ needs.setup.outputs.tag }} | |
| username: ${{ secrets.ACRYL_DOCKER_USERNAME }} | |
| password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} | |
| publish: ${{ needs.setup.outputs.publish }} | |
| context: . | |
| file: ./docker/datahub-ingestion/Dockerfile | |
| platforms: linux/amd64,linux/arm64/v8 | |
| - name: Compute Tag (Full) | |
| id: tag | |
| run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_tag || 'head' }}" >> $GITHUB_OUTPUT | |
| datahub_ingestion_full_scan: | |
| permissions: | |
| contents: read # for actions/checkout to fetch code | |
| security-events: write # for github/codeql-action/upload-sarif to upload SARIF results | |
| actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status | |
| name: "[Monitoring] Scan Datahub Ingestion images for vulnerabilities" | |
| runs-on: ubuntu-latest | |
| needs: [setup, datahub_ingestion_full_build] | |
| steps: | |
| - name: Checkout # adding checkout step just to make trivy upload happy | |
| uses: actions/checkout@v3 | |
| - name: Download image Full Image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.datahub_ingestion_full_build.outputs.needs_artifact_download == 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }} | |
| - name: Run Trivy vulnerability scanner Full Image | |
| uses: aquasecurity/[email protected] | |
| env: | |
| TRIVY_OFFLINE_SCAN: true | |
| with: | |
| image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }} | |
| format: "template" | |
| template: "@/contrib/sarif.tpl" | |
| output: "trivy-results.sarif" | |
| severity: "CRITICAL,HIGH" | |
| ignore-unfixed: true | |
| vuln-type: "os,library" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: "trivy-results.sarif" | |
| smoke_test: | |
| name: Run Smoke Tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_strategy: ["no_cypress", "cypress_suite1", "cypress_rest"] | |
| needs: | |
| [ | |
| setup, | |
| gms_build, | |
| frontend_build, | |
| kafka_setup_build, | |
| mysql_setup_build, | |
| elasticsearch_setup_build, | |
| mae_consumer_build, | |
| mce_consumer_build, | |
| datahub_upgrade_build, | |
| datahub_ingestion_slim_build, | |
| ] | |
| steps: | |
| - name: Disk Check | |
| run: df -h . && docker images | |
| - name: Check out the repo | |
| uses: actions/checkout@v3 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: "zulu" | |
| java-version: 11 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| cache: "pip" | |
| - name: Install dependencies | |
| run: ./metadata-ingestion/scripts/install_deps.sh | |
| - name: Build datahub cli | |
| run: | | |
| ./gradlew :metadata-ingestion:install | |
| - name: Disk Check | |
| run: df -h . && docker images | |
| - name: Remove images | |
| run: docker image prune -a -f || true | |
| - name: Disk Check | |
| run: df -h . && docker images | |
| - name: Download GMS image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Download Frontend image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Download Kafka Setup image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Download Mysql Setup image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Download Elastic Setup image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Download MCE Consumer image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Download MAE Consumer image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Download upgrade image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.setup.outputs.publish != 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }} | |
| - name: Download datahub-ingestion-slim image | |
| uses: ishworkh/docker-image-artifact-download@v1 | |
| if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' }} | |
| with: | |
| image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }} | |
| - name: Disk Check | |
| run: df -h . && docker images | |
| - name: run quickstart | |
| env: | |
| DATAHUB_TELEMETRY_ENABLED: false | |
| DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }} | |
| DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }} | |
| ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag }} | |
| ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5" | |
| ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml" | |
| run: | | |
| ./smoke-test/run-quickstart.sh | |
| - name: sleep 60s | |
| run: | | |
| # we are doing this because gms takes time to get ready | |
| # and we don't have a better readiness check when bootstrap is done | |
| sleep 60s | |
| - name: Disk Check | |
| run: df -h . && docker images | |
| - name: Disable ES Disk Threshold | |
| run: | | |
| curl -XPUT "http://localhost:9200/_cluster/settings" \ | |
| -H 'Content-Type: application/json' -d'{ | |
| "persistent": { | |
| "cluster": { | |
| "routing": { | |
| "allocation.disk.threshold_enabled": false | |
| } | |
| } | |
| } | |
| }' | |
| - name: Remove Source Code | |
| run: find ./*/* ! -path "./metadata-ingestion*" ! -path "./smoke-test*" ! -path "./gradle*" -delete | |
| - name: Disk Check | |
| run: df -h . && docker images | |
| - name: Smoke test | |
| env: | |
| RUN_QUICKSTART: false | |
| DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }} | |
| CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} | |
| CLEANUP_DATA: "false" | |
| TEST_STRATEGY: ${{ matrix.test_strategy }} | |
| run: | | |
| echo "$DATAHUB_VERSION" | |
| ./smoke-test/smoke.sh | |
| - name: Disk Check | |
| run: df -h . && docker images | |
| - name: store logs | |
| if: failure() | |
| run: | | |
| docker ps -a | |
| docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true | |
| docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true | |
| docker logs datahub-mae-consumer >& mae-${{ matrix.test_strategy }}.log || true | |
| docker logs datahub-mce-consumer >& mce-${{ matrix.test_strategy }}.log || true | |
| docker logs broker >& broker-${{ matrix.test_strategy }}.log || true | |
| docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true | |
| docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true | |
| docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true | |
| - name: Upload logs | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: docker logs | |
| path: "*.log" | |
| - name: Upload screenshots | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: cypress-snapshots-${{ matrix.test_strategy }} | |
| path: smoke-test/tests/cypress/cypress/screenshots/ | |
| - uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: Test Results (smoke tests) ${{ matrix.test_strategy }} | |
| path: | | |
| **/build/reports/tests/test/** | |
| **/build/test-results/test/** | |
| **/junit.*.xml | |
| - name: Slack failure notification | |
| if: failure() && github.event_name == 'push' | |
| uses: kpritam/slack-job-status-action@v1 | |
| with: | |
| job-status: ${{ job.status }} | |
| slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }} | |
| channel: github-activities | |
| deploy_datahub_head: | |
| name: Deploy to Datahub HEAD | |
| runs-on: ubuntu-latest | |
| needs: | |
| [ | |
| setup, | |
| smoke_test | |
| ] | |
| steps: | |
| - uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_SQS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SQS_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - uses: isbang/[email protected] | |
| with: | |
| sqs-url: ${{ secrets.DATAHUB_HEAD_SYNC_QUEUE }} | |
| message: '{ "command": "git-sync", "args" : {"repoName": "${{ needs.setup.outputs.repository_name }}", "repoOrg": "${{ github.repository_owner }}", "repoBranch": "${{ needs.setup.outputs.branch_name }}", "repoShaShort": "${{ needs.setup.outputs.short_sha }}" }}' |