Interval Crawler Task #14040
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Interval Crawler Task | |
| on: | |
| schedule: | |
| - cron: '35 8 * 7-9,12 *' | |
| - cron: '35 * * 1-6,10-11 *' | |
| pull_request: | |
| branches: ['main'] | |
| workflow_dispatch: | |
| concurrency: | |
| group: interval-crawler | |
| cancel-in-progress: true | |
| jobs: | |
| crawl-and-push: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| token: ${{ secrets.GH_PAT }} | |
| - name: Pull the latest data branch | |
| run: | | |
| git config user.name ZKLlab-bot | |
| git config user.email [email protected] | |
| git subtree add --prefix=interval-crawler-task-result origin data | |
| - name: Duplicate data folder | |
| run: | | |
| cp -r interval-crawler-task-result data | |
| - name: Set up Python 3.9 | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.9 | |
| - name: Install OpenVPN Client | |
| run: | | |
| sudo apt-get update && sudo apt-get -y install openvpn | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install pipenv | |
| pipenv install | |
| - name: Connect to OpenVPN | |
| run: | | |
| echo "${{ secrets.SHU_USERNAME }}" > /tmp/ovpn-auth.txt | |
| echo "${{ secrets.SHU_PASSWORD }}" >> /tmp/ovpn-auth.txt | |
| sudo openvpn --config ./shu-student.ovpn --auth-user-pass /tmp/ovpn-auth.txt --daemon | |
| - name: Wait for OpenVPN connection | |
| timeout-minutes: 2 | |
| run: | | |
| until ping -c1 xk.autoisp.shu.edu.cn; do sleep 2; done | |
| - name: Setup Node.js environment | |
| uses: actions/[email protected] | |
| with: | |
| node-version: 18.x | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| run: yarn install | |
| - name: Fetch data | |
| env: | |
| SHUSTUID: ${{ secrets.SHU_USERNAME }} | |
| SHUSTUPWD: ${{ secrets.SHU_PASSWORD }} | |
| OUTPUTDIR: interval-crawler-task-result | |
| run: yarn start | |
| - name: Check Results | |
| run: | | |
| tree -I node_modules | |
| - name: Shutdown OpenVPN | |
| run: | | |
| sudo killall openvpn | |
| - name: Compare, push, and create a pull request if necessary | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_PAT }} | |
| run: | | |
| pipenv run python post_crawler.py |