Skip to content

fix(ingest/bigquery): Map BigQuery policy tags to datahub column-leve… #20

fix(ingest/bigquery): Map BigQuery policy tags to datahub column-leve…

fix(ingest/bigquery): Map BigQuery policy tags to datahub column-leve… #20

name: build & test
on:
push:
branches:
- master
paths-ignore:
- "docs/**"
- "**.md"
pull_request:
branches:
- "**"
paths-ignore:
- "docs/**"
- "**.md"
release:
types: [published]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
setup:
runs-on: ubuntu-latest
outputs:
frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }}
ingestion_change: ${{ steps.ci-optimize.outputs.ingestion-change == 'true' }}
backend_change: ${{ steps.ci-optimize.outputs.backend-change == 'true' }}
docker_change: ${{ steps.ci-optimize.outputs.docker-change == 'true' }}
frontend_only: ${{ steps.ci-optimize.outputs.frontend-only == 'true' }}
ingestion_only: ${{ steps.ci-optimize.outputs.ingestion-only == 'true' }}
kafka_setup_change: ${{ steps.ci-optimize.outputs.kafka-setup-change == 'true' }}
mysql_setup_change: ${{ steps.ci-optimize.outputs.mysql-setup-change == 'true' }}
postgres_setup_change: ${{ steps.ci-optimize.outputs.postgres-setup-change == 'true' }}
elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }}
steps:
- name: Check out the repo
uses: acryldata/sane-checkout-action@v3
- uses: ./.github/actions/ci-optimization
id: ci-optimize
build:
strategy:
fail-fast: false
matrix:
command: [
# metadata-ingestion and airflow-plugin each have dedicated build jobs
"except_metadata_ingestion",
"frontend",
]
timezone: ["UTC"]
include:
# We only need the timezone variation for frontend tests.
- command: "frontend"
timezone: "America/New_York"
runs-on: ubuntu-latest
timeout-minutes: 60
needs: setup
steps:
- uses: szenius/[email protected]
with:
timezoneLinux: ${{ matrix.timezone }}
- name: Check out the repo
uses: acryldata/sane-checkout-action@v3
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: 17
- uses: gradle/gradle-build-action@v2
- uses: actions/setup-python@v4
if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
with:
python-version: "3.10"
cache: pip
- name: Gradle build (and test) for NOT metadata ingestion
if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
run: |
./gradlew build \
-x :metadata-ingestion:build \
-x :metadata-ingestion:check \
-x :docs-website:build \
-x :metadata-integration:java:spark-lineage:test \
-x :metadata-io:test \
-x :metadata-ingestion-modules:airflow-plugin:build \
-x :metadata-ingestion-modules:airflow-plugin:check \
-x :metadata-ingestion-modules:dagster-plugin:build \
-x :metadata-ingestion-modules:dagster-plugin:check \
-x :datahub-frontend:build \
-x :datahub-web-react:build \
--parallel
- name: Gradle build (and test) for frontend
if: ${{ matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}
run: |
./gradlew :datahub-frontend:build :datahub-web-react:build --parallel
env:
NODE_OPTIONS: "--max-old-space-size=3072"
- name: Gradle compile (jdk8) for legacy Spark
if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
run: |
./gradlew -PjavaClassVersionDefault=8 :metadata-integration:java:spark-lineage:compileJava
- uses: actions/upload-artifact@v3
if: always()
with:
name: Test Results (build)
path: |
**/build/reports/tests/test/**
**/build/test-results/test/**
**/junit.*.xml
!**/binary/**
- name: Ensure codegen is updated
uses: ./.github/actions/ensure-codegen-updated
quickstart-compose-validation:
runs-on: ubuntu-latest
needs: setup
if: ${{ needs.setup.outputs.docker_change == 'true' }}
steps:
- name: Check out the repo
uses: acryldata/sane-checkout-action@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Quickstart Compose Validation
run: ./docker/quickstart/generate_and_compare.sh
event-file:
runs-on: ubuntu-latest
steps:
- name: Upload
uses: actions/upload-artifact@v3
with:
name: Event File
path: ${{ github.event_path }}