diff --git a/.github/workflows/event-gather-pipeline.yml b/.github/workflows/event-gather-pipeline.yml index fab6fbc..a9f8b2b 100644 --- a/.github/workflows/event-gather-pipeline.yml +++ b/.github/workflows/event-gather-pipeline.yml @@ -46,12 +46,19 @@ jobs: process-events: needs: [deploy-runner-on-gcp] runs-on: [self-hosted, gcp-cdp-runner] + container: + image: ghcr.io/iterative/cml:0-dvc2-base1-gpu + options: --gpus all steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: '3.11' + + - name: Check GPU Drivers + run: | + nvidia-smi - name: Install Packages run: | @@ -63,8 +70,8 @@ jobs: - name: Install Python Dependencies run: | cd python/ + pip install --upgrade pip pip install . - pip install 'faster-whisper @ git+https://github.com/guillaumekln/faster-whisper.git' - name: Setup gcloud uses: google-github-actions/setup-gcloud@v0 @@ -74,10 +81,11 @@ jobs: export_default_credentials: true - name: Dump Credentials to JSON - run: | - echo "$GOOGLE_CREDS" > python/google-creds.json - env: - GOOGLE_CREDS: ${{ secrets.GOOGLE_CREDENTIALS }} + uses: jsdaniell/create-json@v1.2.2 + with: + name: "google-creds.json" + json: ${{ secrets.GOOGLE_CREDENTIALS }} + dir: "python/" - name: Gather and Process New Events - CRON if: ${{ github.event_name == 'schedule' }} diff --git a/.github/workflows/event-index-pipeline.yml b/.github/workflows/event-index-pipeline.yml index e197034..548058f 100644 --- a/.github/workflows/event-index-pipeline.yml +++ b/.github/workflows/event-index-pipeline.yml @@ -43,11 +43,13 @@ jobs: project_id: cdp-milwaukee-9f60e352 service_account_key: ${{ secrets.GOOGLE_CREDENTIALS }} export_default_credentials: true + - name: Dump Credentials to JSON - run: | - echo "$GOOGLE_CREDS" > python/google-creds.json - env: - GOOGLE_CREDS: ${{ secrets.GOOGLE_CREDENTIALS }} + uses: jsdaniell/create-json@v1.2.2 + with: + name: "google-creds.json" + json: ${{ secrets.GOOGLE_CREDENTIALS }} + dir: "python/" # Installs - name: Install Python Dependencies @@ -117,10 +119,11 @@ jobs: service_account_key: ${{ secrets.GOOGLE_CREDENTIALS }} export_default_credentials: true - name: Dump Credentials to JSON - run: | - echo "$GOOGLE_CREDS" > python/google-creds.json - env: - GOOGLE_CREDS: ${{ secrets.GOOGLE_CREDENTIALS }} + uses: jsdaniell/create-json@v1.2.2 + with: + name: "google-creds.json" + json: ${{ secrets.GOOGLE_CREDENTIALS }} + dir: "python/" # Installs - name: Install Python Dependencies diff --git a/.github/workflows/process-special-event.yml b/.github/workflows/process-special-event.yml index 6ed1f9f..2656bb7 100644 --- a/.github/workflows/process-special-event.yml +++ b/.github/workflows/process-special-event.yml @@ -37,12 +37,19 @@ jobs: process-events: needs: [deploy-runner-on-gcp] runs-on: [self-hosted, gcp-cdp-runner] + container: + image: ghcr.io/iterative/cml:0-dvc2-base1-gpu + options: --gpus all steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: '3.11' + + - name: Check GPU Drivers + run: | + nvidia-smi - name: Install Packages run: | @@ -54,8 +61,8 @@ jobs: - name: Install Python Dependencies run: | cd python/ + pip install --upgrade pip pip install . - pip install 'faster-whisper @ git+https://github.com/guillaumekln/faster-whisper.git' - name: Setup gcloud uses: google-github-actions/setup-gcloud@v0 @@ -65,10 +72,11 @@ jobs: export_default_credentials: true - name: Dump Credentials to JSON - run: | - echo "$GOOGLE_CREDS" > python/google-creds.json - env: - GOOGLE_CREDS: ${{ secrets.GOOGLE_CREDENTIALS }} + uses: jsdaniell/create-json@v1.2.2 + with: + name: "google-creds.json" + json: ${{ secrets.GOOGLE_CREDENTIALS }} + dir: "python/" - name: Dump Event Details to JSON run: | diff --git a/.github/workflows/run-script.yml b/.github/workflows/run-script.yml index 0480ef2..798fd5d 100644 --- a/.github/workflows/run-script.yml +++ b/.github/workflows/run-script.yml @@ -38,10 +38,11 @@ jobs: export_default_credentials: true - name: Dump Credentials to JSON - run: | - echo "$GOOGLE_CREDS" > python/google-creds.json - env: - GOOGLE_CREDS: ${{ secrets.GOOGLE_CREDENTIALS }} + uses: jsdaniell/create-json@v1.2.2 + with: + name: "google-creds.json" + json: ${{ secrets.GOOGLE_CREDENTIALS }} + dir: "python/" - name: Run Command if: ${{ github.event_name == 'workflow_dispatch' }} diff --git a/infra/requirements.txt b/infra/requirements.txt index d9dd8ba..29159f2 100644 --- a/infra/requirements.txt +++ b/infra/requirements.txt @@ -1 +1 @@ -cdp-backend==4.0.8 \ No newline at end of file +cdp-backend==4.0.9 \ No newline at end of file diff --git a/python/setup.py b/python/setup.py index d436d11..062ceca 100644 --- a/python/setup.py +++ b/python/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup requirements = [ - "cdp-backend[pipeline]==4.0.8", + "cdp-backend[pipeline]==4.0.9", "cdp-scrapers[milwaukee]", ] diff --git a/web/package.json b/web/package.json index ef9e304..9b88414 100644 --- a/web/package.json +++ b/web/package.json @@ -10,7 +10,7 @@ "deploy": "gh-pages -d build" }, "dependencies": { - "@councildataproject/cdp-frontend": "3.2.0", + "@councildataproject/cdp-frontend": "3.2.3", "react": "^16.13.1", "react-dom": "^16.13.1" },