Refactored redirector script generation #98
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Update redirector yaml config" | |
| # Trigger this workflow on a manual dispatch or a scheduled time | |
| on: | |
| push: | |
| #schedule: | |
| # - cron: '0 * * * *' # Run every hour | |
| workflow_dispatch: # Manually triggered via GitHub Actions UI | |
| env: | |
| GEODB: "/scripts/redirect-config/GeoLite2-City.mmdb" | |
| ASNDB: "/scripts/redirect-config/GeoLite2-ASN.mmdb" | |
| DL_MAP: "/scripts/redirect-config/all-images.json" | |
| SOURCE_OF_TRUTH: "[email protected]:/storage/www/" | |
| concurrency: | |
| group: redirector | |
| cancel-in-progress: false | |
| jobs: | |
| Check: | |
| name: "Check permissions" | |
| runs-on: "ubuntu-24.04" | |
| steps: | |
| - name: "Check permissions" | |
| uses: armbian/actions/team-check@main | |
| with: | |
| ORG_MEMBERS: ${{ secrets.ORG_MEMBERS }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| TEAM: "Release manager" | |
| - name: "Start log" | |
| run: | | |
| echo "# Test summary" >> $GITHUB_STEP_SUMMARY | |
| build: | |
| name: "Get source of truth" | |
| runs-on: ubuntu-24.04 | |
| needs: Check | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| platform: | |
| - beta | |
| - apt | |
| - dl | |
| - archive | |
| steps: | |
| - name: "Install dependencies: lftp" | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: lftp | |
| version: 1.0 | |
| - name: "Download from ${{ env.SOURCE_OF_TRUTH }}${{ matrix.platform }}" | |
| run: | | |
| mkdir -p source destination | |
| if [[ "${{ matrix.platform }}" =~ ^(dl|archive) ]]; then | |
| cd source ; lftp -e "mirror --include-glob=*/archive/*.torrent --parallel=64; exit" https://rsync.armbian.com/${{ matrix.platform }} | |
| cd .. | |
| find source/*/archive/ -mindepth 1 -maxdepth 1 -exec mv -i -- {} destination/ \; | |
| else | |
| cd destination; lftp -e "mirror --parallel=64; exit" https://rsync.armbian.com/${{ matrix.platform }}/dists | |
| fi | |
| - name: "Upload ${{ matrix.platform }} index" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.platform }} | |
| path: destination | |
| compression-level: 0 | |
| retention-days: 2 | |
| Debs-beta-index: | |
| name: "Compare beta deb mirrors" | |
| needs: build | |
| outputs: | |
| matrix: ${{steps.json.outputs.JSON_CONTENT}} | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: "Get beta packages mirrors from database" | |
| id: json | |
| run: | | |
| echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT | |
| curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \ | |
| "${{ secrets.NETBOX_API }}/virtualization/virtual-machines/?limit=500&name__empty=false&tag=debs-beta&status=active" \ | |
| | jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|beta|" | sed "s/\"//g" \ | |
| | xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| Debs-stable-index: | |
| name: "Compare stable deb mirrors" | |
| needs: build | |
| outputs: | |
| matrix: ${{steps.json.outputs.JSON_CONTENT}} | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: "Get beta packages mirrors from database" | |
| id: json | |
| run: | | |
| echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT | |
| curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \ | |
| "${{ secrets.NETBOX_API }}/virtualization/virtual-machines/?limit=500&name__empty=false&tag=debs&status=active" \ | |
| | jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|apt|" | sed "s/\"//g" \ | |
| | xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| Images-stable-index: | |
| name: "Compare stable images mirrors" | |
| needs: build | |
| outputs: | |
| matrix: ${{steps.json.outputs.JSON_CONTENT}} | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: "Get stable images mirrors from database" | |
| id: json | |
| run: | | |
| echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT | |
| curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \ | |
| "${{ secrets.NETBOX_API }}/virtualization/virtual-machines/?limit=500&name__empty=false&tag=images&status=active" \ | |
| | jq '.results[] | .name,.custom_fields["download_path_images"],.id' | sed "s|null|dl|" | sed "s/\"//g" \ | |
| | xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| Images-archive-index: | |
| name: "Compare archive images mirrors" | |
| needs: build | |
| outputs: | |
| matrix: ${{steps.json.outputs.JSON_CONTENT}} | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: "Get archive images mirrors from database" | |
| id: json | |
| run: | | |
| echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT | |
| curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \ | |
| "${{ secrets.NETBOX_API }}/virtualization/virtual-machines/?limit=500&name__empty=false&tag=archive&status=active" \ | |
| | jq '.results[] | .name,.custom_fields["download_path_images"],.id' | sed "s|null|archive|" | sed "s/\"//g" \ | |
| | xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| Debs-beta: | |
| name: "B" | |
| runs-on: ubuntu-24.04 | |
| needs: [ Debs-beta-index ] | |
| outputs: | |
| matrix: ${{needs.Debs-beta-index.outputs.matrix}} | |
| if: ${{ needs.Debs-beta-index.outputs.matrix != '[]' && needs.Debs-beta-index.outputs.matrix != '' }} | |
| timeout-minutes: 6 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node: ${{fromJson(needs.Debs-beta-index.outputs.matrix)}} | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: beta | |
| path: debs-beta | |
| - name: "Install dependencies" | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: lftp | |
| version: 1.0 | |
| - name: "Check ${{ matrix.node }} " | |
| run: | | |
| SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1) | |
| SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2) | |
| echo "SERVER_ID=${SERVER_ID}" >> $GITHUB_ENV | |
| mkdir -p compare; cd compare | |
| if curl --output /dev/null --silent --head --fail "https://${SERVER_URL}/dists/"; then | |
| timeout 5m lftp -e "mirror --parallel=16; exit" https://${SERVER_URL}/dists/ || true | |
| fi | |
| cd .. | |
| OUT=$(diff -rq compare debs || true) | |
| mkdir -p status | |
| if [[ -z "${OUT}" ]]; then | |
| echo "true" >> status/${SERVER_ID} | |
| echo "STATUS=true" >> $GITHUB_ENV | |
| elif [[ "${exit_status}" -eq 0 ]]; then | |
| echo "not_in_sync" >> status/${SERVER_ID} | |
| echo "STATUS=not_in_sync" >> $GITHUB_ENV | |
| elif [[ "${exit_status}" -eq 124 ]]; then | |
| echo "timeout" >> status/${SERVER_ID} | |
| echo "STATUS=not_in_sync" >> $GITHUB_ENV | |
| fi | |
| - name: Upload ${{ env.STATUS }} for ${{ matrix.node }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: debs-beta-${{ env.SERVER_ID }} | |
| path: status | |
| if-no-files-found: ignore | |
| Debs-stable: | |
| name: "S" | |
| runs-on: ubuntu-24.04 | |
| needs: [ Debs-stable-index ] | |
| outputs: | |
| matrix: ${{needs.Debs-stable-index.outputs.matrix}} | |
| if: ${{ needs.Debs-stable-index.outputs.matrix != '[]' && needs.Debs-stable-index.outputs.matrix != '' }} | |
| timeout-minutes: 12 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node: ${{fromJson(needs.Debs-stable-index.outputs.matrix)}} | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: apt | |
| path: debs | |
| - name: "Install dependencies" | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: lftp | |
| version: 1.0 | |
| - name: "Check ${{ matrix.node }} " | |
| run: | | |
| SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1) | |
| SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2) | |
| echo "SERVER_ID=${SERVER_ID}" >> $GITHUB_ENV | |
| mkdir -p compare; cd compare | |
| if curl --output /dev/null --silent --head --fail "https://${SERVER_URL}/dists/"; then | |
| timeout 11m lftp -e "mirror --parallel=16; exit" https://${SERVER_URL}/dists/ || exit_status=$? | |
| fi | |
| cd .. | |
| OUT=$(diff -rq compare debs || true) | |
| mkdir -p status | |
| if [[ -z "${OUT}" ]]; then | |
| echo "true" >> status/${SERVER_ID} | |
| echo "STATUS=true" >> $GITHUB_ENV | |
| elif [[ "${exit_status}" -eq 0 ]]; then | |
| echo "not_in_sync" >> status/${SERVER_ID} | |
| echo "STATUS=not_in_sync" >> $GITHUB_ENV | |
| elif [[ "${exit_status}" -eq 124 ]]; then | |
| echo "timeout" >> status/${SERVER_ID} | |
| echo "STATUS=not_in_sync" >> $GITHUB_ENV | |
| fi | |
| - name: Upload ${{ env.STATUS }} for ${{ matrix.node }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: debs-${{ env.SERVER_ID }} | |
| path: status | |
| if-no-files-found: ignore | |
| Images-stable: | |
| name: "I" | |
| runs-on: ubuntu-24.04 | |
| needs: [ Images-stable-index ] | |
| outputs: | |
| matrix: ${{needs.Images-stable-index.outputs.matrix}} | |
| if: ${{ needs.Images-stable-index.outputs.matrix != '[]' && needs.Images-stable-index.outputs.matrix != '' }} | |
| timeout-minutes: 12 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node: ${{fromJson(needs.Images-stable-index.outputs.matrix)}} | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: dl | |
| path: dl | |
| - name: "Install dependencies" | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: lftp | |
| version: 1.0 | |
| - name: "Check ${{ matrix.node }} " | |
| run: | | |
| SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1) | |
| SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2) | |
| echo "SERVER_ID=${SERVER_ID}" >> $GITHUB_ENV | |
| mkdir -p compare source; cd source | |
| if curl --output /dev/null --silent --head --fail "https://${SERVER_URL}"; then | |
| timeout 11m lftp -e "mirror --include-glob=*/archive/*.torrent --parallel=64; exit" https://${SERVER_URL} || exit_status=$? | |
| cd .. | |
| find source/. -mindepth 3 -exec mv -i -- {} compare/ \; | |
| fi | |
| OUT=$(diff -rq compare dl || true) | |
| mkdir -p status | |
| if [[ -z "${OUT}" ]]; then | |
| echo "true" >> status/${SERVER_ID} | |
| echo "STATUS=true" >> $GITHUB_ENV | |
| elif [[ "${exit_status}" -eq 0 ]]; then | |
| echo "not_in_sync" >> status/${SERVER_ID} | |
| echo "STATUS=not_in_sync" >> $GITHUB_ENV | |
| elif [[ "${exit_status}" -eq 124 ]]; then | |
| echo "timeout" >> status/${SERVER_ID} | |
| echo "STATUS=not_in_sync" >> $GITHUB_ENV | |
| fi | |
| - name: Upload ${{ env.STATUS }} for ${{ matrix.node }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dl-${{ env.SERVER_ID }} | |
| path: status | |
| if-no-files-found: ignore | |
| Images-archive: | |
| name: "I" | |
| runs-on: ubuntu-24.04 | |
| needs: [ Images-archive-index ] | |
| outputs: | |
| matrix: ${{needs.Images-archive-index.outputs.matrix}} | |
| if: ${{ needs.Images-archive-index.outputs.matrix != '[]' && needs.Images-archive-index.outputs.matrix != '' }} | |
| timeout-minutes: 12 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node: ${{fromJson(needs.Images-archive-index.outputs.matrix)}} | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: archive | |
| path: archive | |
| - name: "Install dependencies" | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: lftp | |
| version: 1.0 | |
| - name: "Check ${{ matrix.node }} " | |
| run: | | |
| SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1) | |
| SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2) | |
| echo "SERVER_ID=${SERVER_ID}" >> $GITHUB_ENV | |
| mkdir -p compare source; cd source | |
| if curl --output /dev/null --silent --head --fail "https://${SERVER_URL}"; then | |
| timeout 11m lftp -e "mirror --include-glob=*/archive/*.torrent --parallel=64; exit" https://${SERVER_URL} || exit_status=$? | |
| cd .. | |
| find source/. -mindepth 3 -exec mv -i -- {} compare/ \; | |
| fi | |
| OUT=$(diff -rq compare archive || true) | |
| mkdir -p status | |
| if [[ -z "${OUT}" ]]; then | |
| echo "true" >> status/${SERVER_ID} | |
| echo "STATUS=true" >> $GITHUB_ENV | |
| elif [[ "${exit_status}" -eq 0 ]]; then | |
| echo "not_in_sync" >> status/${SERVER_ID} | |
| echo "STATUS=not_in_sync" >> $GITHUB_ENV | |
| elif [[ "${exit_status}" -eq 124 ]]; then | |
| echo "timeout" >> status/${SERVER_ID} | |
| echo "STATUS=not_in_sync" >> $GITHUB_ENV | |
| fi | |
| - name: Upload ${{ env.STATUS }} for ${{ matrix.node }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: archive-${{ env.SERVER_ID }} | |
| path: status | |
| if-no-files-found: ignore | |
| download: | |
| needs: [ Debs-beta,Debs-stable,Images-stable,Images-archive ] | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Download All Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: status | |
| pattern: debs-beta-* | |
| merge-multiple: true | |
| - name: "Grep only those that are in sync" | |
| run: | | |
| echo "# Not in sync" >> $GITHUB_STEP_SUMMARY | |
| echo "$(grep not_in_sync status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY | |
| echo "# Timeouts" >> $GITHUB_STEP_SUMMARY | |
| echo "$(grep timeout status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY | |
| echo "failoverserver=$(grep true status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV | |
| echo "reloadKey=$(openssl rand -hex 16)" >> $GITHUB_ENV | |
| - name: Test debs | |
| uses: armbian/actions/make-yaml-redirector@redirector | |
| with: | |
| variant: debs-beta | |
| failoverserver: "${{ env.failoverserver }}" | |
| port: 8083 | |
| geodb: "${{ env.GEODB }}" | |
| asndb: "${{ env.ASNDB }}" | |
| dl_map: "${{ env.DL_MAP }}" | |
| reloadKey: ${{ env.reloadKey }} | |
| netbox: ${{ secrets.NETBOX_TOKEN }} | |
| - name: Download All Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: status | |
| pattern: debs* | |
| merge-multiple: true | |
| - name: "Grep only those that are in sync" | |
| run: | | |
| echo "# Not in sync" >> $GITHUB_STEP_SUMMARY | |
| echo "$(grep not_in_sync status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY | |
| echo "# Timeouts" >> $GITHUB_STEP_SUMMARY | |
| echo "$(grep timeout status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY | |
| echo "failoverserver=$(grep true status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV | |
| echo "reloadKey=$(openssl rand -hex 16)" >> $GITHUB_ENV | |
| - name: Test debs | |
| uses: armbian/actions/make-yaml-redirector@redirector | |
| with: | |
| variant: debs | |
| failoverserver: "${{ env.failoverserver }}" | |
| port: 8083 | |
| geodb: "${{ env.GEODB }}" | |
| asndb: "${{ env.ASNDB }}" | |
| dl_map: "${{ env.DL_MAP }}" | |
| reloadKey: ${{ env.reloadKey }} | |
| netbox: ${{ secrets.NETBOX_TOKEN }} | |
| - name: Download All Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: status | |
| pattern: dl* | |
| merge-multiple: true | |
| - name: "Grep only those that are in sync" | |
| run: | | |
| echo "# Not in sync" >> $GITHUB_STEP_SUMMARY | |
| echo "$(grep not_in_sync status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY | |
| echo "# Timeouts" >> $GITHUB_STEP_SUMMARY | |
| echo "$(grep timeout status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY | |
| echo "failoverserver=$(grep true status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV | |
| echo "reloadKey=$(openssl rand -hex 16)" >> $GITHUB_ENV | |
| - name: Test debs | |
| uses: armbian/actions/make-yaml-redirector@redirector | |
| with: | |
| variant: images | |
| failoverserver: "${{ env.failoverserver }}" | |
| port: 8083 | |
| geodb: "${{ env.GEODB }}" | |
| asndb: "${{ env.ASNDB }}" | |
| dl_map: "${{ env.DL_MAP }}" | |
| reloadKey: ${{ env.reloadKey }} | |
| netbox: ${{ secrets.NETBOX_TOKEN }} | |
| Close: | |
| needs: [ download ] | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| path: armbian.github.io | |
| - uses: robinraju/[email protected] | |
| with: | |
| repository: "P3TERX/GeoLite.mmdb" | |
| tag: '2024.12.10' | |
| fileName: "*.mmdb" | |
| - name: Download All Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: status | |
| pattern: config-* | |
| merge-multiple: true | |
| - name: "Check" | |
| run: | | |
| ls -l status | |
| echo "-" | |
| ls -l ${{ github.workspace }}/ | |
| - name: Commit changes if any | |
| run: | | |
| cd armbian.github.io | |
| git checkout data | |
| mkdir -p data/ | |
| cp ${{ github.workspace }}/status/*.yaml data/ | |
| cp ${{ github.workspace }}/*.mmdb data/ | |
| git config --global user.name "github-actions" | |
| git config --global user.email "[email protected]" | |
| git add data/. | |
| git diff --cached --quiet || git commit -m "Update Redirector YAML configuration" | |
| git push | |
| - uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| name: | | |
| debs-* | |
| dl-* | |
| apt | |
| beta | |
| dl | |
| archive |