Scrape and publish 174 #174
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scrape and publish | |
| run-name: Scrape and publish ${{ github.run_number }} | |
| on: | |
| schedule: | |
| - cron: "0 18 * * 1-5" | |
| workflow_dispatch: | |
| jobs: | |
| generate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Checkout pp-data repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: havasd/pp-data | |
| path: pp-data | |
| - uses: getsentry/[email protected] | |
| id: venv | |
| with: | |
| python-version: 3.11 | |
| cache-dependency-path: | | |
| requirements.txt | |
| requirements-frozen.txt | |
| install-cmd: pip install -r requirements.txt -c requirements-frozen.txt | |
| - name: Scrape historical prices | |
| run: | | |
| cd price_scraper | |
| scrapy crawl aranykor -a base_dir=../pp-data -L INFO | |
| scrapy crawl mak -a base_dir=../pp-data -L INFO | |
| scrapy crawl bamosz -a base_dir=../pp-data -L INFO | |
| scrapy crawl otp_nyugdij -a base_dir=../pp-data -L INFO | |
| scrapy crawl budapest_nyugdij -a base_dir=../pp-data -L INFO | |
| scrapy crawl mbh_nyugdij -a base_dir=../pp-data -L INFO | |
| scrapy crawl horizont_nyugdij -a base_dir=../pp-data -L INFO | |
| scrapy crawl allianz_nyugdij -a base_dir=../pp-data -L INFO | |
| scrapy crawl pannonia_nyugdij -a base_dir=../pp-data -L INFO | |
| scrapy crawl erste_nyugdij -a base_dir=../pp-data -L INFO | |
| scrapy crawl alfa_nyugdij -a base_dir=../pp-data -L INFO | |
| scrapy crawl honved_nyugdij -a base_dir=../pp-data -L INFO | |
| - name: Push historical prices to pp-data | |
| uses: cpina/github-action-push-to-another-repository@main | |
| env: | |
| SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }} | |
| API_TOKEN_GITHUB: ${{ secrets.API_TOKEN_GITHUB }} | |
| with: | |
| source-directory: 'pp-data' | |
| commit-message: 'Update pricing data from ORIGIN_COMMIT' | |
| destination-github-username: 'havasd' | |
| destination-repository-name: 'pp-data' | |
| user-email: [email protected] | |
| target-branch: main |