process-special-event.yml

name: Process Special Event

on:
  workflow_dispatch:
    inputs:
      event-details-json-string:
        description: "Event details (see EventIngestionModel in cdp-backend) in JSON string form. The string must be free of any newline or tab characters."
        required: true

permissions:
  id-token: write
  contents: write
  pull-requests: write

jobs:
  deploy-runner-on-gcp:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Setup CML
        uses: iterative/setup-cml@v1
      - name: Create Runner
        env:
          REPO_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
          GOOGLE_APPLICATION_CREDENTIALS_DATA: ${{ secrets.GOOGLE_CREDENTIALS }}
        run: |
          cml runner \
            --single \
            --labels=gcp-cdp-runner \
            --cloud=gcp \
            --cloud-region=us-central1-f \
            --cloud-type=n1-standard-4 \
            --cloud-gpu=nvidia-tesla-t4 \
            --cloud-hdd-size=30 \
            --idle-timeout=600

  process-events:
    needs: [deploy-runner-on-gcp]
    runs-on: [self-hosted, gcp-cdp-runner]
    container:
      image: ghcr.io/iterative/cml:0-dvc2-base1-gpu
      options: --gpus all

    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: '3.11'
    
    - name: Check GPU Drivers
      run: |
        nvidia-smi

    - name: Install Packages
      run: |
        sudo apt update
        sudo apt-get install -y --no-install-recommends \
          libsndfile1 \
          ffmpeg

    - name: Install Python Dependencies
      run: |
        cd python/
        pip install --upgrade pip
        pip install .

    - name: Setup gcloud
      uses: google-github-actions/setup-gcloud@v0
      with:
        project_id: cdp-milwaukee-9f60e352
        service_account_key: ${{ secrets.GOOGLE_CREDENTIALS }}
        export_default_credentials: true

    - name: Dump Credentials to JSON
      uses: jsdaniell/create-json@v1.2.2
      with:
        name: "google-creds.json"
        json: ${{ secrets.GOOGLE_CREDENTIALS }}
        dir: "python/"

    - name: Dump Event Details to JSON
      run: |
        echo ${{ github.event.inputs.event-details-json-string }} > python/event-details.json

    - name: Process special events into event gather pipeline
      if: ${{ github.event_name == 'workflow_dispatch' }}
      run: |
        cd python/
        process_special_event \
          --event_details_file event-details.json \
          --event_gather_config_file event-gather-config.json