Skip to content
Update redirector YAML config

Redirector update #247

Sign in to view logs

Redirector update

Redirector update #247

Workflow file for this run

.github/workflows/generate-redirector-config.yml at b90261b
name: "Update redirector YAML config"
# Trigger this workflow on a manual dispatch or a scheduled time
on:
repository_dispatch:
types: ["Redirector update"]
schedule:
- cron: '0 * * * *' # Run every hour
workflow_dispatch: # Manually triggered via GitHub Actions UI
env:
GEODB: "/scripts/redirect-config/GeoLite2-City.mmdb"
ASNDB: "/scripts/redirect-config/GeoLite2-ASN.mmdb"
DL_MAP: "/scripts/redirect-config/all-images.json"
SOURCE_OF_TRUTH: "[email protected]:/storage/www/"
concurrency:
group: redirector
cancel-in-progress: false
jobs:
Check:
name: "Check permissions"
runs-on: "ubuntu-24.04"
steps:
- name: "Check permissions"
uses: armbian/actions/team-check@main
with:
ORG_MEMBERS: ${{ secrets.ORG_MEMBERS }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TEAM: "Release manager"
build:
name: "Get source of truth"
runs-on: ubuntu-24.04
needs: Check
strategy:
fail-fast: true
matrix:
platform:
- beta
- apt
- dl
- archive
steps:
- name: "Install dependencies: lftp"
uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: lftp
version: 1.0
- name: "Download from ${{ env.SOURCE_OF_TRUTH }}${{ matrix.platform }}"
run: |
mkdir -p source destination
if [[ "${{ matrix.platform }}" =~ ^(dl|archive) ]]; then
cd source ; lftp -e "mirror --include-glob=*/archive/*.torrent --parallel=64; exit" https://rsync.armbian.com/${{ matrix.platform }}
cd ..
find source/*/archive/ -mindepth 1 -maxdepth 1 -exec mv -i -- {} destination/ \;
else
cd destination; lftp -e "mirror --parallel=64; exit" https://rsync.armbian.com/${{ matrix.platform }}/dists
fi
- name: "Upload ${{ matrix.platform }} index"
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.platform }}
path: destination
compression-level: 0
retention-days: 2
Debs-beta-index:
name: "Compare beta deb mirrors"
needs: build
outputs:
matrix: ${{steps.json.outputs.JSON_CONTENT}}
runs-on: ubuntu-24.04
steps:
- name: "Get beta packages mirrors from database"
id: json
run: |
echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \
"${{ secrets.NETBOX_API }}/virtualization/virtual-machines/?limit=500&name__empty=false&tag=debs-beta&status=active" \
| jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|beta|" | sed "s/\"//g" \
| xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT
echo 'EOF' >> $GITHUB_OUTPUT
Debs-stable-index:
name: "Compare stable deb mirrors"
needs: build
outputs:
matrix: ${{steps.json.outputs.JSON_CONTENT}}
runs-on: ubuntu-24.04
steps:
- name: "Get beta packages mirrors from database"
id: json
run: |
echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \
"${{ secrets.NETBOX_API }}/virtualization/virtual-machines/?limit=500&name__empty=false&tag=debs&status=active" \
| jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|apt|" | sed "s/\"//g" \
| xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT
echo 'EOF' >> $GITHUB_OUTPUT
Images-stable-index:
name: "Compare stable images mirrors"
needs: build
outputs:
matrix: ${{steps.json.outputs.JSON_CONTENT}}
runs-on: ubuntu-24.04
steps:
- name: "Get stable images mirrors from database"
id: json
run: |
echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \
"${{ secrets.NETBOX_API }}/virtualization/virtual-machines/?limit=500&name__empty=false&tag=images&status=active" \
| jq '.results[] | .name,.custom_fields["download_path_images"],.id' | sed "s|null|dl|" | sed "s/\"//g" \
| xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT
echo 'EOF' >> $GITHUB_OUTPUT
Images-archive-index:
name: "Compare archive images mirrors"
needs: build
outputs:
matrix: ${{steps.json.outputs.JSON_CONTENT}}
runs-on: ubuntu-24.04
steps:
- name: "Get archive images mirrors from database"
id: json
run: |
echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \
"${{ secrets.NETBOX_API }}/virtualization/virtual-machines/?limit=500&name__empty=false&tag=archive&status=active" \
| jq '.results[] | .name,.custom_fields["download_path_images"],.id' | sed "s|null|archive|" | sed "s/\"//g" \
| xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT
echo 'EOF' >> $GITHUB_OUTPUT
Debs-beta:
name: "B"
runs-on: ubuntu-24.04
needs: [ Debs-beta-index ]
outputs:
matrix: ${{needs.Debs-beta-index.outputs.matrix}}
if: ${{ needs.Debs-beta-index.outputs.matrix != '[]' && needs.Debs-beta-index.outputs.matrix != '' }}
timeout-minutes: 7
strategy:
fail-fast: false
matrix:
node: ${{fromJson(needs.Debs-beta-index.outputs.matrix)}}
steps:
- uses: actions/download-artifact@v4
with:
name: beta
path: debs-beta
- name: "Install dependencies"
uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: lftp
version: 1.0
- name: "Check ${{ matrix.node }} "
run: |
SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1)
SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2)
echo "SERVER_ID=${SERVER_ID}" >> $GITHUB_ENV
mkdir -p compare; cd compare
if curl --output /dev/null --silent --head --fail "https://${SERVER_URL}/dists/"; then
timeout 5m lftp -e "mirror --parallel=16; exit" https://${SERVER_URL}/dists/ || true
fi
cd ..
OUT=$(diff -rq compare debs || true)
mkdir -p status
if [[ -z "${OUT}" ]]; then
echo "true" >> status/${SERVER_ID}
echo "STATUS=true" >> $GITHUB_ENV
elif [[ "${exit_status}" -eq 0 ]]; then
echo "not_in_sync" >> status/${SERVER_ID}
echo "${SERVER_URL}" >> status/${SERVER_ID}
echo "STATUS=not_in_sync" >> $GITHUB_ENV
elif [[ "${exit_status}" -eq 124 ]]; then
echo "timeout" >> status/${SERVER_ID}
echo "${SERVER_URL}" >> status/${SERVER_ID}
echo "STATUS=not_in_sync" >> $GITHUB_ENV
fi
- name: Upload ${{ env.STATUS }} for ${{ matrix.node }}
uses: actions/upload-artifact@v4
with:
name: debs-beta-${{ env.SERVER_ID }}
path: status
if-no-files-found: ignore
Debs-stable:
name: "S"
runs-on: ubuntu-24.04
needs: [ Debs-stable-index ]
outputs:
matrix: ${{needs.Debs-stable-index.outputs.matrix}}
if: ${{ needs.Debs-stable-index.outputs.matrix != '[]' && needs.Debs-stable-index.outputs.matrix != '' }}
timeout-minutes: 7
strategy:
fail-fast: false
matrix:
node: ${{fromJson(needs.Debs-stable-index.outputs.matrix)}}
steps:
- uses: actions/download-artifact@v4
with:
name: apt
path: debs
- name: "Install dependencies"
uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: lftp
version: 1.0
- name: "Check ${{ matrix.node }} "
run: |
SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1)
SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2)
echo "SERVER_ID=${SERVER_ID}" >> $GITHUB_ENV
mkdir -p compare; cd compare
if curl --output /dev/null --silent --head --fail "https://${SERVER_URL}/dists/"; then
timeout 5m lftp -e "mirror --parallel=16; exit" https://${SERVER_URL}/dists/ || exit_status=$?
fi
cd ..
OUT=$(diff -rq compare debs || true)
mkdir -p status
if [[ -z "${OUT}" ]]; then
echo "true" >> status/${SERVER_ID}
echo "STATUS=true" >> $GITHUB_ENV
elif [[ "${exit_status}" -eq 0 ]]; then
echo "not_in_sync" >> status/${SERVER_ID}
echo "${SERVER_URL}" >> status/${SERVER_ID}
echo "STATUS=not_in_sync" >> $GITHUB_ENV
elif [[ "${exit_status}" -eq 124 ]]; then
echo "timeout" >> status/${SERVER_ID}
echo "${SERVER_URL}" >> status/${SERVER_ID}
echo "STATUS=not_in_sync" >> $GITHUB_ENV
fi
- name: Upload ${{ env.STATUS }} for ${{ matrix.node }}
uses: actions/upload-artifact@v4
with:
name: debs-${{ env.SERVER_ID }}
path: status
if-no-files-found: ignore
Images-stable:
name: "I"
runs-on: ubuntu-24.04
needs: [ Images-stable-index ]
outputs:
matrix: ${{needs.Images-stable-index.outputs.matrix}}
if: ${{ needs.Images-stable-index.outputs.matrix != '[]' && needs.Images-stable-index.outputs.matrix != '' }}
timeout-minutes: 7
strategy:
fail-fast: false
matrix:
node: ${{fromJson(needs.Images-stable-index.outputs.matrix)}}
steps:
- uses: actions/download-artifact@v4
with:
name: dl
path: dl
- name: "Install dependencies"
uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: lftp
version: 1.0
- name: "Check ${{ matrix.node }} "
run: |
SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1)
SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2)
echo "SERVER_ID=${SERVER_ID}" >> $GITHUB_ENV
mkdir -p compare source; cd source
if curl --output /dev/null --silent --head --fail "https://${SERVER_URL}"; then
timeout 5m lftp -e "mirror --include-glob=*/archive/*.torrent --parallel=64; exit" https://${SERVER_URL} || exit_status=$?
cd ..
find source/*/archive/ -mindepth 1 -maxdepth 1 -exec mv -i -- {} compare/ \; || true
fi
OUT=$(diff -rq compare dl || true)
mkdir -p status
if [[ -z "${OUT}" ]]; then
echo "true" >> status/${SERVER_ID}
echo "STATUS=true" >> $GITHUB_ENV
elif [[ "${exit_status}" -eq 0 ]]; then
echo "not_in_sync" >> status/${SERVER_ID}
echo "${SERVER_URL}" >> status/${SERVER_ID}
echo "STATUS=not_in_sync" >> $GITHUB_ENV
elif [[ "${exit_status}" -eq 124 ]]; then
echo "timeout" >> status/${SERVER_ID}
echo "${SERVER_URL}" >> status/${SERVER_ID}
echo "STATUS=not_in_sync" >> $GITHUB_ENV
fi
- name: Upload ${{ env.STATUS }} for ${{ matrix.node }}
uses: actions/upload-artifact@v4
with:
name: dl-${{ env.SERVER_ID }}
path: status
if-no-files-found: ignore
Images-archive:
name: "A"
runs-on: ubuntu-24.04
needs: [ Images-archive-index ]
outputs:
matrix: ${{needs.Images-archive-index.outputs.matrix}}
if: ${{ needs.Images-archive-index.outputs.matrix != '[]' && needs.Images-archive-index.outputs.matrix != '' }}
timeout-minutes: 7
strategy:
fail-fast: false
matrix:
node: ${{fromJson(needs.Images-archive-index.outputs.matrix)}}
steps:
- uses: actions/download-artifact@v4
with:
name: archive
path: archive
- name: "Install dependencies"
uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: lftp
version: 1.0
- name: "Check ${{ matrix.node }} "
run: |
SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1)
SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2)
echo "SERVER_ID=${SERVER_ID}" >> $GITHUB_ENV
mkdir -p compare source; cd source
if curl --output /dev/null --silent --head --fail "https://${SERVER_URL}"; then
timeout 5m lftp -e "mirror --include-glob=*/archive/*.torrent --parallel=64; exit" https://${SERVER_URL} || exit_status=$?
cd ..
find source/*/archive/ -mindepth 1 -maxdepth 1 -exec mv -i -- {} compare/ \; || true
fi
OUT=$(diff -rq compare archive || true)
mkdir -p status
if [[ -z "${OUT}" ]]; then
echo "true" >> status/${SERVER_ID}
echo "STATUS=true" >> $GITHUB_ENV
elif [[ "${exit_status}" -eq 0 ]]; then
echo "not_in_sync" >> status/${SERVER_ID}
echo "${SERVER_URL}" >> status/${SERVER_ID}
echo "STATUS=not_in_sync" >> $GITHUB_ENV
elif [[ "${exit_status}" -eq 124 ]]; then
echo "timeout" >> status/${SERVER_ID}
echo "${SERVER_URL}" >> status/${SERVER_ID}
echo "STATUS=not_in_sync" >> $GITHUB_ENV
fi
- name: Upload ${{ env.STATUS }} for ${{ matrix.node }}
uses: actions/upload-artifact@v4
with:
name: archive-${{ env.SERVER_ID }}
path: status
if-no-files-found: ignore
download:
needs: [ Debs-beta,Debs-stable,Images-stable,Images-archive ]
runs-on: ubuntu-24.04
steps:
- name: Download All Artifacts
uses: actions/download-artifact@v4
with:
path: status
pattern: debs-beta-*
merge-multiple: true
- name: "Start log"
run: |
echo "# Test summary" >> $GITHUB_STEP_SUMMARY
- name: "Grep only those that are in sync"
run: |
echo "# Not in sync" >> $GITHUB_STEP_SUMMARY
#echo "$(grep not_in_sync status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY
grep not_in_sync status/* | cut -d":" -f1 | xargs awk 'FNR==2{print}' >> $GITHUB_STEP_SUMMARY
echo "# Timeouts" >> $GITHUB_STEP_SUMMARY
grep timeout status/* | cut -d":" -f1 | xargs awk 'FNR==2{print}' >> $GITHUB_STEP_SUMMARY
#echo "$(grep timeout status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY
echo "failoverserver=$(grep true status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV
echo "reloadKey=$(openssl rand -hex 16)" >> $GITHUB_ENV
rm -rf status
- name: Test debs
uses: armbian/actions/make-yaml-redirector@main
with:
variant: debs-beta
failoverserver: "${{ env.failoverserver }}"
port: 8083
geodb: "${{ env.GEODB }}"
asndb: "${{ env.ASNDB }}"
dl_map: "${{ env.DL_MAP }}"
reloadKey: ${{ env.reloadKey }}
netbox: ${{ secrets.NETBOX_TOKEN }}
- name: Download All Artifacts
uses: actions/download-artifact@v4
with:
path: status
pattern: debs*
merge-multiple: true
- name: "Grep only those that are in sync"
run: |
echo "# Not in sync" >> $GITHUB_STEP_SUMMARY
#echo "$(grep not_in_sync status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY
grep not_in_sync status/* | cut -d":" -f1 | xargs awk 'FNR==2{print}' >> $GITHUB_STEP_SUMMARY
echo "# Timeouts" >> $GITHUB_STEP_SUMMARY
grep timeout status/* | cut -d":" -f1 | xargs awk 'FNR==2{print}' >> $GITHUB_STEP_SUMMARY
#echo "$(grep timeout status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY
echo "failoverserver=$(grep true status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV
echo "reloadKey=$(openssl rand -hex 16)" >> $GITHUB_ENV
rm -rf status
- name: Stable debs
uses: armbian/actions/make-yaml-redirector@main
with:
variant: debs
failoverserver: "${{ env.failoverserver }}"
geodb: "${{ env.GEODB }}"
asndb: "${{ env.ASNDB }}"
dl_map: "${{ env.DL_MAP }}"
reloadKey: ${{ env.reloadKey }}
netbox: ${{ secrets.NETBOX_TOKEN }}
- name: Download All Artifacts
uses: actions/download-artifact@v4
with:
path: status
pattern: dl*
merge-multiple: true
- name: "Grep only those that are in sync"
run: |
echo "# Not in sync" >> $GITHUB_STEP_SUMMARY
#echo "$(grep not_in_sync status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY
grep not_in_sync status/* | cut -d":" -f1 | xargs awk 'FNR==2{print}' >> $GITHUB_STEP_SUMMARY
echo "# Timeouts" >> $GITHUB_STEP_SUMMARY
grep timeout status/* | cut -d":" -f1 | xargs awk 'FNR==2{print}' >> $GITHUB_STEP_SUMMARY
#echo "$(grep timeout status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY
echo "failoverserver=$(grep true status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV
echo "reloadKey=$(openssl rand -hex 16)" >> $GITHUB_ENV
rm -rf status
- name: Images
uses: armbian/actions/make-yaml-redirector@main
with:
variant: images
failoverserver: "${{ env.failoverserver }}"
port: 8081
geodb: "${{ env.GEODB }}"
asndb: "${{ env.ASNDB }}"
dl_map: "${{ env.DL_MAP }}"
reloadKey: ${{ env.reloadKey }}
netbox: ${{ secrets.NETBOX_TOKEN }}
- name: Download All Artifacts
uses: actions/download-artifact@v4
with:
path: status
pattern: archive*
merge-multiple: true
- name: "Grep only those that are in sync"
run: |
echo "# Not in sync" >> $GITHUB_STEP_SUMMARY
#echo "$(grep not_in_sync status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY
grep not_in_sync status/* | cut -d":" -f1 | xargs awk 'FNR==2{print}' >> $GITHUB_STEP_SUMMARY
echo "# Timeouts" >> $GITHUB_STEP_SUMMARY
grep timeout status/* | cut -d":" -f1 | xargs awk 'FNR==2{print}' >> $GITHUB_STEP_SUMMARY
#echo "$(grep timeout status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_STEP_SUMMARY
echo "failoverserver=$(grep true status/* | cut -d":" -f1 | cut -d"/" -f2 | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV
echo "reloadKey=$(openssl rand -hex 16)" >> $GITHUB_ENV
rm -rf status
- name: Archive
uses: armbian/actions/make-yaml-redirector@main
with:
variant: archive
failoverserver: "${{ env.failoverserver }}"
port: 8082
geodb: "${{ env.GEODB }}"
asndb: "${{ env.ASNDB }}"
dl_map: "${{ env.DL_MAP }}"
reloadKey: ${{ env.reloadKey }}
netbox: ${{ secrets.NETBOX_TOKEN }}
Close:
needs: [ download ]
runs-on: ubuntu-24.04
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
path: armbian.github.io
- uses: robinraju/[email protected]
with:
repository: "P3TERX/GeoLite.mmdb"
fileName: "*.mmdb"
latest: true
- name: Download All Artifacts
uses: actions/download-artifact@v4
with:
path: status
pattern: config-*
merge-multiple: true
- name: "Check"
run: |
ls -l status
echo "-"
ls -l ${{ github.workspace }}/
- name: Commit changes if any
run: |
cd armbian.github.io
git checkout data
mkdir -p data/
cp ${{ github.workspace }}/status/*.yaml data/
cp ${{ github.workspace }}/*.mmdb data/
git config --global user.name "github-actions"
git config --global user.email "[email protected]"
git add data/.
git diff --cached --quiet || git commit -m "Update Redirector YAML configuration"
git push
- uses: geekyeggo/delete-artifact@v5
with:
name: |
debs-*
dl-*
archive-*
apt
beta
dl
archive
- name: "Run webindex update action"
uses: peter-evans/repository-dispatch@v3
with:
token: ${{ secrets.GITHUB_TOKEN }}
event-type: "Webindex update"