Scrape Releases #4289
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scrape Releases | |
| on: | |
| push: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: "0 */8 * * *" | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| scrape: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: 🛒 Checkout the repo | |
| uses: actions/checkout@v4 | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| cache: 'pip' | |
| - run: pip install -r requirements.txt | |
| - name: 🔖 Get latest Mihomo (stable) version | |
| id: mihomo-version | |
| run: | | |
| echo "stable=$(curl -sL "https://api.github.com/repos/MetaCubeX/mihomo/releases/latest" | jq -r '.tag_name')" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: 🍱 Restore SQLite Database from Cache | |
| id: cache-db-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: releases.db | |
| key: releases_db | |
| - name: 🍱 Restore Mihomo Installer from Cache | |
| id: cache-mihomo-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}.deb | |
| key: mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }} | |
| - name: 🤿 Setup and start Mihomo | |
| run: | | |
| # Download Mihomo Installer if needed | |
| [ ! -f "mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}.deb" ] && curl -fsSL "https://github.com/MetaCubeX/mihomo/releases/download/${{ steps.mihomo-version.outputs.stable }}/mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}.deb" -o "mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}.deb" | |
| # Install Mihomo | |
| sudo dpkg -i ./mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}.deb | |
| # Run Mihomo with our configuration (as daemon) | |
| ./mihomo_daemon.py | |
| shell: bash | |
| - name: 🕸️ Run crawler | |
| run: | | |
| scrapy crawl releases -L DEBUG --logfile="scrapy.log" | |
| shell: bash | |
| - name: 🥡 Save SQLite Database to Cache | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: releases.db | |
| key: ${{ steps.cache-db-restore.outputs.cache-primary-key }} | |
| - name: 🥡 Save Mihomo Installer to Cache | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: mihomo-linux-amd64-${{ steps.mihomo-version.outputs.stable }}.deb | |
| key: ${{ steps.cache-mihomo-restore.outputs.cache-primary-key }} | |
| - name: ✅ Validate stuff | |
| run: | | |
| [ -s "${{ github.workspace }}/releases.csv" ] && echo "🎉 CSV file generated successfully!" || exit 1 | |
| [ -s "${{ github.workspace }}/releases.json" ] && echo "🎉 JSON file generated successfully!" || exit 1 | |
| [ -s "${{ github.workspace }}/releases.jsonl" ] && echo "🎉 JSONLines file generated successfully!" || exit 1 | |
| [ -s "${{ github.workspace }}/releases.rss" ] && echo "🎉 RSS Feed generated successfully!" || exit 1 | |
| [ -s "${{ github.workspace }}/releases.db" ] && echo "🎉 SQLite DB updated successfully!" || exit 1 | |
| shell: bash | |
| - name: 🗒️ Upload Scrapy logs | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: scrapy.log | |
| path: ${{ github.workspace }}/scrapy.log | |
| - name: 🗒️ Upload Mihomo logs | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: mihomo.log | |
| path: ${{ github.workspace }}/mihomo.log | |
| - name: ✏️ Sort items | |
| run: | | |
| ./sort_items.py && echo "🎉 Items sorted successfully!" || exit 1 | |
| shell: bash | |
| - name: 🚮 Delete older releases | |
| uses: dev-drprasad/[email protected] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| keep_latest: 3 | |
| delete_tags: true | |
| - name: 🛠️ Generate stats | |
| id: gen-stats | |
| run: | | |
| # Date | |
| DATE="$(date -u '+%Y%m%d%H%M%S')" | |
| echo "date=$DATE" >> $GITHUB_OUTPUT | |
| # Number of releases | |
| RELEASES="$(jq length "${{ github.workspace }}/releases.json")" | |
| echo "releases=$RELEASES" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: 🚀 Upload new release | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| tag_name: feeds-${{ steps.gen-stats.outputs.date }} | |
| name: ${{ steps.gen-stats.outputs.date }} | |
| body: | | |
| Number of releases - ${{ steps.gen-stats.outputs.releases }} | |
| files: | | |
| ${{ github.workspace }}/releases.csv | |
| ${{ github.workspace }}/releases.json | |
| ${{ github.workspace }}/releases.jsonl | |
| ${{ github.workspace }}/releases.rss | |
| ${{ github.workspace }}/releases.db | |
| draft: false | |
| prerelease: false |