Skip to content

Dataflow Integration Test #1174

Dataflow Integration Test

Dataflow Integration Test #1174

Workflow file for this run

name: Dataflow Integration Test
on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]
types: [ opened, reopened, synchronize, labeled ]
schedule:
- cron: '0 4 * * *' # run once a day at 4 AM
jobs:
test-dataflow:
# run on:
# - all pushes to main
# - schedule defined above
# - a PR was just labeled 'test-dataflow' or 'test-all'
# - a PR with 'test-dataflow' or 'test-all' label was opened, reopened, or synchronized
if: |
github.event_name == 'push' ||
github.event_name == 'schedule' ||
github.event.label.name == 'test-all' ||
github.event.label.name == 'test-dataflow' ||
contains( github.event.pull_request.labels.*.name, 'test-all') ||
contains( github.event.pull_request.labels.*.name, 'test-dataflow')
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.9", "3.10", "3.11"]
recipes-version: [
"pangeo-forge-recipes==0.10.4",
]
beam-version: [
"apache-beam[gcp]==2.52.0",
]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- name: 'Authenticate to Google Cloud'
uses: 'google-github-actions/auth@v1'
with:
credentials_json: '${{ secrets.GCP_DATAFLOW_SERVICE_KEY }}'
- name: Install dependencies & our package
run: |
python -m pip install --upgrade pip
python -m pip install -e .[test]
- name: 'Run Dataflow Integration Test'
run: |
pytest -vvvxs tests/integration/test_dataflow_integration.py \
--recipes-version=${{ matrix.recipes-version }} \
--beam-version=${{ matrix.beam-version }}