scrape-test.yml

name: Quick scrape test
on:
  push:
jobs:
  unit-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - run: pip3 install -r requirements.dev.txt
      - run: PYTHONPATH=. pytest -v tests
  scrape-woolies:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - run: pip3 install -r requirements.txt
      - run: python3 main.py sync --quick woolies
      - uses: actions/upload-artifact@v3
        with:
          name: woolies_snapshot
          path: ./output/woolies/
  scrape-coles:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - run: pip3 install -r requirements.txt
      - run: python3 main.py sync --quick coles
      - uses: actions/upload-artifact@v3
        with:
          name: coles_snapshot
          path: ./output/coles/
  merge-price-history:
    permissions:
      contents: read  # Required for checkout action
      id-token: write # This is required for requesting the JWT
    runs-on: ubuntu-latest
    needs:
      - scrape-woolies
      - scrape-coles
    steps:
      - uses: actions/checkout@v4
      - name: Download coles artifact
        uses: actions/download-artifact@v3
        with:
          name: coles_snapshot
          path: ./output/coles/
      - name: Download woolies artifact
        uses: actions/download-artifact@v3
        with:
          name: woolies_snapshot
          path: ./output/woolies/
      - run: pip3 install -r requirements.txt
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: "${{ vars.TARGET_ROLE_ARN }}"
          aws-region: ap-southeast-2
      # Use "cp" because it is an error if the file doesn't exist. It means history
      # is broken and needs to be built from scratch locally.
      - run: aws s3 cp s3://grocery-scrape-au/latest-canonical.json.gz ./output/
      - run: python3 main.py analysis
      - uses: actions/upload-artifact@v3
        with:
          name: latest_canonical
          path: ./output/latest-canonical.json.gz