daily_job_board_scrape.yml

name: Daily Python Script

on:
  schedule:
    - cron: "0 7 * * *"
  workflow_dispatch:

jobs:
  run_script:
    name: Call Scrapy Script
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Run Scrapy script
        run: |
          cd job_board_scraper
          python run_job_scraper.py
          python get_ashby_jobs.py
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          HASHIDS_SALT: ${{ secrets.HASHIDS_SALT }}
          PAGES_TO_SCRAPE_QUERY: ${{ secrets.PAGES_TO_SCRAPE_QUERY }}
          ASHBY_PAGES_TO_SCRAPE_QUERY: ${{ secrets.ASHBY_PAGES_TO_SCRAPE_QUERY }}
          PG_DATABASE: ${{ secrets.PG_DATABASE }}
          PG_HOST: ${{ secrets.PG_HOST }}
          PG_PASSWORD: ${{ secrets.PG_PASSWORD }}
          PG_USER: ${{ secrets.PG_USER }}
          CHUNK_SIZE: ${{ secrets.CHUNK_SIZE }}
          RAW_HTML_S3_BUCKET: ${{ secrets.RAW_HTML_S3_BUCKET }}

  run_rippling_script:
    name: Call Rippling Script
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Run Rippling Script
        run: |
          cd job_board_scraper
          python get_rippling_jobs.py rippling
        env:
          HASHIDS_SALT: ${{ secrets.HASHIDS_SALT }}
          PG_DATABASE: ${{ secrets.PG_DATABASE }}
          PG_HOST: ${{ secrets.PG_HOST }}
          PG_PASSWORD: ${{ secrets.PG_PASSWORD }}
          PG_USER: ${{ secrets.PG_USER }}
          PG_PORT: ${{ secrets.PG_PORT }}
          GET_BOARD_TOKENS_BASE_QUERY: ${{ secrets.GET_BOARD_TOKENS_BASE_QUERY }}
          RIPPLING_JOBS_OUTLINE_TABLE_NAME: ${{ secrets.RIPPLING_JOBS_OUTLINE_TABLE_NAME }}

  run_dbt:
    name: Run dbt
    needs: [run_script, run_rippling_script]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Install dbt dependencies
        run: cd levergreen_dbt && dbt deps

      - name: Build dbt models
        run: cd levergreen_dbt && dbt run
        env:
          PG_DATABASE: ${{ secrets.PG_DATABASE }}
          PG_HOST: ${{ secrets.PG_HOST }}
          PG_PASSWORD: ${{ secrets.PG_PASSWORD }}
          PG_USER: ${{ secrets.PG_USER }}