Merge d0a9a35 into c5626c4

.github/workflows/build_and_publish_dev.yaml

@@ -1,55 +1,14 @@
-name: build-and-publish-dev
+name: Build and Publish Dev Images
 on:
   push:
+    branches-ignore: [master, main]
 jobs:
   main-ci:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Fetch tag history
         run: git fetch --tags
-      - uses: google-github-actions/[email protected]
-        name: Setup gcloud for Dataflow tests
-        with:
-          project_id: ${{ secrets.DEV_PROJECT_ID }}
-          service_account_key: ${{ secrets.GCP_TEST_KEY }}
-          export_default_credentials: true
-      - name: Set up Python 3.9 for dataflow tests
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9.16
-      - name: Install Poetry
-        uses: snok/[email protected]
-        with:
-          version: 1.1.9
-      - name: Restore cache dependencies
-        uses: actions/cache@v2
-        env:
-          cache-name: cache-poetry-v2
-        with:
-          path: ~/.cache/pypoetry
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('./orchestration/pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-build-${{ env.cache-name }}-
-            ${{ runner.os }}-build-
-            ${{ runner.os }}-
-      - name: Install dependencies
-        run: poetry install
-        working-directory: ${{ github.workspace }}/orchestration
-      - uses: olafurpg/setup-scala@v10
-        with:
-          java-version: [email protected]
-      - name: Check Scala formatting
-        run: sbt scalafmtCheckAll
-      - name: Scala Compile
-        run: sbt Compile/compile Test/compile IntegrationTest/compile
-      - name: Scala Test
-        run: sbt "set ThisBuild/coverageEnabled := true" test IntegrationTest/test coverageAggregate
-      - name: Run E2E test suite
-        run: poetry run pytest -v -m e2e
-        working-directory: ${{ github.workspace }}/orchestration
-      - name: Publish Scala coverage
-        uses: codecov/codecov-action@v1
       - uses: google-github-actions/[email protected]
         name: Setup gcloud for pushing Docker images
         with:
@@ -58,6 +17,9 @@ jobs:
           export_default_credentials: true
       - name: Setup GCR auth
         run: gcloud auth configure-docker --quiet us.gcr.io,us-east4-docker.pkg.dev
+      - uses: olafurpg/setup-scala@v10
+        with:
+          java-version: [email protected]
       - name: Push Scala Dataflow Docker image
         run: sbt publish
       - name: Get artifact slug
@@ -75,4 +37,3 @@ jobs:
           context: .
           push: true
           tags: us-east4-docker.pkg.dev/broad-dsp-monster-hca-dev/monster-dev-env/hca_ingest_compose_dev_env:${{steps.get-artifact-slug.outputs.slug}}, us-east4-docker.pkg.dev/broad-dsp-monster-hca-dev/monster-dev-env/hca_ingest_compose_dev_env:dev
-

.github/workflows/build-and-publish-main.yaml → .github/workflows/build_and_publish_main.yaml

@@ -1,10 +1,11 @@
-name: Main Validation and Release
+name: Main Validation and Release Workflow
 on:
-  push:
-    branches:
-      - main
+  pull_request_target:
+      types:
+        - closed
 jobs:
   main-ci:
+    if: github.event.pull_request.merged == true
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

.github/workflows/validate-pull-request.yaml → .github/workflows/validate_pull_request_main.yaml

@@ -1,10 +1,11 @@
-name: PR Validation
+name: PR Validation for Main
 on:
   pull_request:
     branches:
       - main
 jobs:
   pr-validation:
+    name: PR Validation
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

.gitignore

@@ -17,3 +17,5 @@ dagster_home/
 venv/
 # Build files
 build/
+# Test files
+*_manifest.csv

orchestration/hca_manage/manifest.py

@@ -40,12 +40,16 @@
     "dev": "broad-dsp-monster-hca-dev-etl-partitions",
     "prod": "broad-dsp-monster-hca-prod-etl-partitions"
 }
+# Test contains a single staging area for testing purposes
+# The staging area is not used for any production pipelines
+# Be sure to delete any snapshots and datasets created using this test staging area
 STAGING_AREA_BUCKETS = {
     "prod": {
         "EBI": "gs://broad-dsp-monster-hca-prod-ebi-storage/prod",
         "UCSC": "gs://broad-dsp-monster-hca-prod-ebi-storage/prod",
         "LANTERN": "gs://broad-dsp-monster-hca-prod-lantern",
-        "LATTICE": "gs://broad-dsp-monster-hca-prod-lattice/staging"
+        "LATTICE": "gs://broad-dsp-monster-hca-prod-lattice/staging",
+        "TEST": "gs://broad-dsp-monster-hca-prod-ebi-storage/broad_test_dataset"
     },
     "dev": {
         "EBI": "gs://broad-dsp-monster-hca-dev-ebi-staging/dev",