Webindex update #96
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update download JSON index | |
| on: | |
| repository_dispatch: | |
| types: ["Webindex update"] | |
| concurrency: | |
| group: redirector | |
| cancel-in-progress: false | |
| jobs: | |
| Webindex: | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| name: "Generate JSON Index" | |
| runs-on: "ubuntu-24.04" | |
| steps: | |
| - name: Checkout build framework repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: armbian/build | |
| fetch-depth: 1 | |
| clean: false | |
| path: build | |
| - name: Checkout OS repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: armbian/os | |
| fetch-depth: 1 | |
| clean: false | |
| path: os | |
| - name: Checkout armbian.github.io repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: armbian/armbian.github.io | |
| fetch-depth: 0 | |
| clean: false | |
| path: armbian.github.io | |
| - name: Install SSH key | |
| uses: shimataro/ssh-key-action@v2 | |
| with: | |
| key: ${{ secrets.KEY_UPLOAD }} | |
| known_hosts: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }} | |
| if_key_exists: replace | |
| - name: "Install dependencies" | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: gh jc zip | |
| version: 1.0 | |
| - name: "Generate map file" | |
| run: | | |
| # make sure its cleaned | |
| rm -f exposed.map.autoproposal | |
| # get all stable files we provide | |
| rsync -e "ssh -p 10023 -o StrictHostKeyChecking=accept-new" -ar [email protected]:/storage/www/dl/ | awk '{ print $5 }' | grep "archive/" | grep -v ".txt\|.xz.\|homeassistant\|openhab\|kali\|omv" | grep ".xz" > /tmp/stable.txt | |
| # generate map entries | |
| # $1 = board | |
| # $2 = branch | |
| function check_targets(){ | |
| b=$(echo $1 | rev | cut -d"/" -f1 | rev | cut -d"." -f1) | |
| I=0 | |
| for bs in $(cat /tmp/stable.txt | grep ${b}/); do | |
| GNOME=$(echo $bs | grep "$2" | grep _gnome | grep jammy | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)_/_[0-9].*/' | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)/_[0-9]*\.[0-9]*\.[0-9]*/') | |
| [[ -n $GNOME ]] && echo $GNOME && I=$(( I + 1 )) | |
| XFCE=$(echo $bs | grep $2 | grep _xfce | grep jammy | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)_/_[0-9].*/' | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)/_[0-9]*\.[0-9]*\.[0-9]*/') | |
| [[ -n $XFCE ]] && echo $XFCE && I=$(( I + 1 )) | |
| SERVER=$(echo $bs | grep $2 | grep -v "minimal\|desktop" | grep bookworm | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)_/_[0-9].*/' | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)/_[0-9]*\.[0-9]*\.[0-9]*/' ) | |
| [[ -n ${SERVER} ]] && echo $SERVER && I=$(( I + 1 )) | |
| [[ $I -eq 2 ]] && break | |
| done | |
| JAMMYCLI=$(cat /tmp/stable.txt | grep $b | grep $2 | grep -v "minimal\|desktop" | grep jammy | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)_/_[0-9].*/' | sed 's/[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)/_[0-9]*\.[0-9]*\.[0-9]*/' ) | |
| [[ $I -lt 2 && -n $JAMMYCLI ]] && echo "$JAMMYCLI" | |
| return $I | |
| } | |
| for board in $(ls -1 build/config/boards/*.{conf,csc,wip,tvb}); do | |
| # read kernel target | |
| if [[ $(grep KERNEL_TARGET -w $board | cut -d '"' -f 2 | sed "s/,/ /g" | wc -w) -gt 2 ]]; then | |
| KERNEL_TARGET=$(grep KERNEL_TARGET -w $board | cut -d '"' -f 2 | awk -F, '{ print $1,$3,$2 }') | |
| else | |
| KERNEL_TARGET=$(grep KERNEL_TARGET -w $board | cut -d '"' -f 2 | sed "s/,/ /g") | |
| fi | |
| VARIANTS=0 | |
| for targets in ${KERNEL_TARGET}; do | |
| if [[ "${targets}" == current && $VARIANTS -lt 2 ]]; then | |
| check_targets "$board" "current" | |
| VARIANTS=$((VARIANTS+$?)) | |
| fi | |
| if [[ "${targets}" == legacy && $VARIANTS -lt 2 ]]; then | |
| check_targets "$board" "legacy" | |
| VARIANTS=$((VARIANTS+$?)) | |
| fi | |
| if [[ "${targets}" == edge && $VARIANTS -lt 2 ]]; then | |
| check_targets "$board" "edge" | |
| VARIANTS=$((VARIANTS+$?)) | |
| fi | |
| done || true | |
| done | tee -a > exposed.map.autoproposal | |
| # | |
| # Lets fix this and go back to manual in case of troubles | |
| # cp exposed.map.autoproposal os/exposed.map | |
| - name: "Build JSON file" # adjust "exposed" property while reading from download | |
| run: | | |
| # prepare feeds | |
| #A=$(rsync -ar rsync://rsync.armbian.com/dl/ | awk '{ print ""$2"|https://dl.armbian.com/"$5"|"$3"T"$4"Z" }' | grep "archive/" | grep -v ".txt") | |
| A=$(rsync -e "ssh -p 10023 -o StrictHostKeyChecking=accept-new" -ar [email protected]:/storage/www/dl/ | awk '{ print ""$2"|https://dl.armbian.com/"$5"|"$3"T"$4"Z" }' | grep "archive/" | grep -v ".txt\|homeassistant\|openhab\|kali\|omv") | |
| B=$(gh release view --json assets --repo github.com/armbian/community | jq '.assets[] | .size, .url, .createdAt' | xargs -n3 -d'\n' | grep -v ".txt" | sed "s/\"//g" | sed -e 's| |\||g') | |
| C=$(gh release view --json assets --repo github.com/armbian/os | jq '.assets[] | .size, .url, .createdAt' | xargs -n3 -d'\n' | grep -v ".txt" | sed "s/\"//g" | sed -e 's| |\||g') | |
| D=$(gh release view --json assets --repo github.com/armbian/distribution | jq '.assets[] | .size, .url, .createdAt' | xargs -n3 -d'\n' | grep -v ".txt" | sed "s/\"//g" | sed -e 's| |\||g') | |
| # debug | |
| echo "$A" > a.txt | |
| echo "$B" > b.txt | |
| echo "$C" > c.txt | |
| echo "$D" > d.txt | |
| # join all in one file and compress | |
| for line in $A $B $C $D; do | |
| BOARD=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f2) | |
| ARMBIAN_VERSION=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f1) | |
| IMAGE_SIZE=$(echo $line | cut -d"|" -f1 | sed "s/\.//g" | sed "s/\,//g") | |
| KERNEL_BRANCH=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f4) | |
| IMAGE_RELEASE=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f3) | |
| IMAGE_URL=$(echo $line | cut -d"|" -f2) | |
| # workarounds | |
| STEP_A=1 | |
| STEP_B=2 | |
| [[ $IMAGE_URL == *i3-wm* || $IMAGE_URL == *kde-* ]] && STEP_A=2 && STEP_B=3 | |
| IMAGE_TARGET=$(echo $IMAGE_URL | cut -d"|" -f2 | grep "minimal\|desktop" | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f4- | grep -Po '_[a-z].*' | cut -d"." -f1 | sed "s/_//" | sed "s/_desktop//" | cut -d"-" -f1,${STEP_A}) | |
| IMAGE_EXTENSION=$(echo $IMAGE_URL | cut -d"|" -f2 | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | sed 's/_sm8250-xiaomi-umi\|_sm8250-xiaomi-elish//g' | sed "s/rc[0-9]//g" | cut -d"_" -f4- | cut -d"-" -f${STEP_B}- | cut -d"_" -f1 | cut -d"." -f1) | |
| [[ $IMAGE_EXTENSION == $KERNEL_BRANCH || $IMAGE_EXTENSION == ${IMAGE_TARGET} || $IMAGE_EXTENSION =~ boot|csot|boe|sms ]] && unset IMAGE_EXTENSION | |
| [[ -z ${IMAGE_TARGET} ]] && IMAGE_TARGET="server" | |
| FILE_EXTENSION=$(echo $IMAGE_URL | cut -d"|" -f2 | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | sed 's/_sm8250-xiaomi-umi\|_sm8250-xiaomi-elish//g' | sed "s/-rc[0-9]//g" | rev | cut -d"_" -f1 | rev | sed 's/.*[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)//' | sed "s/desktop.\|minimal.//") | |
| #FILE_EXTENSION=$(echo $IMAGE_URL | grep -o "oowow.*\|img.*") | |
| # Clean out application from extension | |
| FILE_EXTENSION=$(echo $FILE_EXTENSION | sed 's/.*-'$IMAGE_EXTENSION'//g' | sed -e 's/^\.//g') | |
| IMAGE_TYPE=$(echo $IMAGE_URL | cut -d"/" -f5) | |
| IMAGE_CREATED=$(echo $line | cut -d"|" -f3 | sed "s/\//-/g") | |
| EXPOSED=false | |
| # Assemble redirector link | |
| PREFIX="" | |
| [[ "${IMAGE_TYPE}" == "os" ]] && PREFIX="nightly/" | |
| REDI_EXT=$(echo $FILE_EXTENSION | rev | cut -d"." -f1 | rev | sed "s/xz//g") | |
| REDI_URL="https://dl.armbian.com/${PREFIX}${BOARD,,}/${IMAGE_RELEASE^}_${KERNEL_BRANCH}${IMAGE_TARGET:+_$IMAGE_TARGET}${IMAGE_EXTENSION:+-$IMAGE_EXTENSION}${REDI_EXT:+.$REDI_EXT}" | |
| # bypass redirector for weirdos | |
| if [[ "${FILE_EXTENSION}" =~ boot|csot|boe|sms|rootfs|qcow2 ]]; then REDI_URL=${IMAGE_URL}; fi | |
| # check if image is exposed | |
| while read exposed; do | |
| [[ $IMAGE_URL =~ $exposed ]] && EXPOSED=true | |
| done < os/exposed.map | |
| echo "${BOARD,,}|$ARMBIAN_VERSION|$IMAGE_URL|$REDI_URL|$IMAGE_CREATED|$IMAGE_SIZE|$IMAGE_RELEASE|$KERNEL_BRANCH|$IMAGE_TARGET|$IMAGE_EXTENSION|$EXPOSED|$IMAGE_TYPE|$FILE_EXTENSION" | |
| done | sed '1s/^/"board_slug"|"armbian_version"|"file_url"|"redi_url"|"file_updated"|"file_size"|"distro_release"|"kernel_branch"|"image_variant"|"preinstalled_application"|"promoted"|"download_repository"|"file_extension"\n/' | jc --csv | jq '{"assets": .}' > all-images.json | |
| - name: "Download and compress torrent files" | |
| run: | | |
| SOURCE=$(mktemp -d) | |
| DESTINATION=$(mktemp -d) | |
| rsync -e "ssh -p 10023 -o StrictHostKeyChecking=accept-new" -zqvr --include="*/archive/*.torrent" --exclude="/*/*/*" --exclude="_*/" --exclude="control" --exclude="quotes.txt" --exclude="*/all-torrents.zip" [email protected]:/storage/www/dl/ ${SOURCE} | |
| find ${SOURCE}/. -mindepth 3 -exec mv -i -- {} ${DESTINATION}/ \; | |
| zip -qj all-torrents.zip ${DESTINATION}/*.torrent | |
| - name: Commit changes if any | |
| run: | | |
| cd armbian.github.io | |
| git checkout data | |
| mkdir -p data/ | |
| cp ${{ github.workspace }}/all-torrents.zip data/ | |
| cp ${{ github.workspace }}/all-images.json data/ | |
| cp ${{ github.workspace }}/exposed.map.autoproposal data/ | |
| cp ${{ github.workspace }}/os/kernel-description.json data/ | |
| git config --global user.name "github-actions" | |
| git config --global user.email "[email protected]" | |
| git add data/. | |
| git diff --cached --quiet || git commit -m "Update WEB indes files" | |
| git push | |
| - name: "Run Jira update action" | |
| uses: peter-evans/repository-dispatch@v3 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| event-type: "Jira update" |