Skip to content

Cron

Cron #453

Workflow file for this run

name: Cron
on:
schedule:
- cron: "27 6 * * *"
workflow_dispatch:
env:
CI: true
PIPENV_VENV_IN_PROJECT: true
SCRAPY_SETTINGS_MODULE: city_scrapers.settings.prod
WAYBACK_ENABLED: true
AUTOTHROTTLE_MAX_DELAY: 30.0
AUTOTHROTTLE_START_DELAY: 1.5
AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0
AZURE_ACCOUNT_KEY: ${{ secrets.AZURE_ACCOUNT_KEY }}
AZURE_ACCOUNT_NAME: ${{ secrets.AZURE_ACCOUNT_NAME }}
AZURE_CONTAINER: ${{ secrets.AZURE_CONTAINER }}
AZURE_STATUS_CONTAINER: ${{ secrets.AZURE_STATUS_CONTAINER }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
GOOGLE_CLOUD_API_KEY: ${{ secrets.GOOGLE_CLOUD_API_KEY }}
jobs:
crawl:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.9
uses: actions/setup-python@v1
with:
python-version: 3.9
- name: Install Pipenv
uses: dschep/install-pipenv-action@v1
- name: Cache Python dependencies
uses: actions/cache@v1
with:
path: .venv
key: pip-3.9-${{ hashFiles('**/Pipfile.lock') }}
restore-keys: |
pip-3.9-
pip-
- name: Install dependencies
run: pipenv sync
env:
PIPENV_DEFAULT_PYTHON_VERSION: 3.9
- name: Run scrapers
run: |
export PYTHONPATH=$(pwd):$PYTHONPATH
./.deploy.sh
- name: Combine output feeds
run: |
export PYTHONPATH=$(pwd):$PYTHONPATH
pipenv run scrapy combinefeeds -s LOG_ENABLED=True
- name: Prevent workflow deactivation
uses: gautamkrishnar/keepalive-workflow@v1
with:
committer_username: "citybureau-bot"
committer_email: "[email protected]"