ci: add multi branch build

lime-system: add apk option in 92_add-lime-repos ci: update feeds scheme to build C++ packages for 3 lastest openwrt branches the new scheme make these packages available at https://github.com/libremesh/lime-feed referenced by apk in /etc/apk/repositories as @libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/openwrt_main/x86_64/packages.adb ci: disable sign packages in main openwrt branch not yet supported, there is an active pull Support sign apk feed openwrt/gh-action-sdk#46 to install packages use "apk add --allow-untrusted lime-docs"
libremesh · Nov 17, 2024 · 1ec656c · 1ec656c
1 parent de3bd5f
commit 1ec656c
Show file tree

Hide file tree

Showing 4 changed files with 187 additions and 75 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -12,36 +12,52 @@ on:
       - ".github/workflows/*"
       - "libremesh.mk"
       - "packages/**"
+  workflow_dispatch:
 
 jobs:
+# Build all packages for x86_64
+# Use openwrt branches 'main' and 'openwrt-24.10' to produce two feeds:
+# one for 'apk' and one for the older 'opkg'.
+# NOTE: this doesn't sign packages for apk
   build:
-    name: Build ${{ github.ref }}
+    name: Build ${{ matrix.version }} ${{ github.ref }}
     runs-on: ubuntu-latest
-
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        include:
+          - { version: main, output_path: 'apk/'}
+          - { version: openwrt-24.10, output_path: ''}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Build packages
-        uses: openwrt/gh-action-sdk@v5
+        uses: openwrt/gh-action-sdk@v7
         env:
-          ARCH: "x86_64"
+          ARCH: "x86_64-${{ matrix.version }}"
           FEEDNAME: "libremesh"
           IGNORE_ERRORS: "n m y"
-          KEY_BUILD: "${{ secrets.KEY_BUILD }}"
-
+          KEY_BUILD: "${{ matrix.version != 'main' && secrets.KEY_BUILD || '' }}"
+          INDEX: 1
+          V: s
+          NO_DEFAULT_FEEDS: 1
+          NO_REFRESH_CHECK: 1
+          NO_SHFMT_CHECK: 1
+
       - name: Set package destination
         run: |
           export TAG=$(echo "${{ github.ref }}" | cut -d '/' -f 3- | perl -pe 's/v([0-9])/$1/')
           echo "$TAG"
           echo "DEST_DIR=$TAG" >> $GITHUB_ENV
 
       - name: Upload packages to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
-          external_repository: libremesh/lime-feed
+          external_repository: a-gave/lime-feed
           publish_dir: bin/packages/x86_64/libremesh/
-          destination_dir: ${{ env.DEST_DIR }}
+          destination_dir: ${{ matrix.output_path }}${{ env.DEST_DIR }}
 
 #     - name: Upload packages to S3
 #       uses: jakejarvis/s3-sync-action@master

diff --git a/.github/workflows/multi-arch-build.yml b/.github/workflows/multi-arch-build.yml
@@ -14,74 +14,130 @@ on:
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
+# Build packages using the openwrt sdk
+# for each architecture supported
+# in openwrt branchs 'main', 'stable', 'old-stable'
+# 
+# NOTE: 
+# This list of branches will be updated manually 
+# - at next openwrt release, ideally adding '25.xx' and removing '23.05'
+# - whether it will be identified an incompatibility of the current libremesh code with an openwrt branch
+env:
+  BUILD_BRANCHES: "main 24.10 23.05"
+
+# workaround: 'make package/shared-state-async/check' fails with 'Package HASH check failed'
+  PACKAGES: "shared-state-bat_hosts"
+
 jobs:
+  generate_matrix:
+    name: 'Generate matrix'
+    runs-on: ubuntu-latest
+    outputs:
+      matrix_builds: ${{ steps.define_matrix.outputs.matrix_builds }}
+    steps:
+      - name: Define matrix of branchs and archs
+        id: define_matrix
+        run: |
+          JSON='['
+          FIRST_BUILD=1
+          echo "${{ env.BUILD_BRANCHES }}"
+          for version in ${{ env.BUILD_BRANCHES }}; do
+
+            VERSION=$([ "$version" == "main" ] \
+              && echo "main" || echo "openwrt-$version")
+            echo $VERSION
+
+            VERSION_PATH=$([ "$version" == "main" ] \
+              && echo "snapshots/packages" || echo "releases/packages-$version")
+            echo $VERSION_PATH
+
+            OUTPUT_PATH=$([ "$version" == "main" ] \
+              && echo "openwrt_main" || echo "openwrt_$version")
+            echo $OUTPUT_PATH
+
+            wget -r -nH --no-parent --level 1 --accept html -P. \
+              --directory-prefix=. https://downloads.openwrt.org/$VERSION_PATH/  
+
+            for arch in $(ls ./$VERSION_PATH/ | grep -v html); do
+              [[ $FIRST_BUILD -ne 1 ]] && JSON="$JSON"','
+              FIRST_BUILD=0
+
+              JSON="$JSON"'{"version": "'"$VERSION"'" ,"arch": "'"$arch"'", "output_path": "'"$OUTPUT_PATH"'" }'
+              echo $JSON
+            done
+          done
+
+          matrix_include='{"include": '"$JSON"']}'
+          echo "matrix_builds=${matrix_include}" >> "$GITHUB_OUTPUT"
+          
   build:
-    name: build ${{ matrix.arch }} ${{ github.ref }}
+    name: build ${{ matrix.arch }}-${{ matrix.version }}
     runs-on: ubuntu-latest
+    needs: generate_matrix
     strategy:
       fail-fast: false
-      max-parallel: 1
-      matrix:
-        arch:
-          - aarch64_cortex-a53
-          - aarch64_cortex-a72
-          - aarch64_generic
-          - arm_arm1176jzf-s_vfp
-          - arm_arm926ej-s
-          - arm_cortex-a15_neon-vfpv4
-          - arm_cortex-a5_vfpv4
-          - arm_cortex-a7
-          - arm_cortex-a7_neon-vfpv4
-          - arm_cortex-a7_vfpv4
-          - arm_cortex-a8_vfpv3
-          - arm_cortex-a9
-          - arm_cortex-a9_neon
-          - arm_cortex-a9_vfpv3-d16
-          - arm_fa526
-          - arm_mpcore
-          - arm_xscale
-          - i386_pentium-mmx
-          - i386_pentium4
-          - mips64_octeonplus
-          - mips_24kc
-          - mips_4kec
-          - mips_mips32
-          - mipsel_24kc
-          - mipsel_24kc_24kf
-          - mipsel_74kc
-          - mipsel_mips32
-          - powerpc_464fp
-          - powerpc_8548
-          - riscv64_riscv64
-          - x86_64
-
+      max-parallel: 10
+      matrix: ${{ fromJson(needs.generate_matrix.outputs.matrix_builds) }}
+      # matrix:
+      #   version: ['main']
+      #   arch: ['x86_64']
+      #   output_path: ['openwrt_main']
     steps:
       - uses: actions/checkout@v4
 
-      - name: Build packages ${{ matrix.arch }}
+      - name: Build packages ${{ matrix.arch }}-${{ matrix.version }}
         uses: openwrt/gh-action-sdk@v7
         env:
-          ARCH: "${{ matrix.arch }}"
+          ARCH: "${{ matrix.arch }}-${{ matrix.version }}"
           FEEDNAME: "libremesh"
           IGNORE_ERRORS: "n m y"
-          KEY_BUILD: "${{ secrets.KEY_BUILD }}"
-          PACKAGES: "shared-state-bat_hosts"
+          KEY_BUILD: "${{ matrix.version != 'main' && secrets.KEY_BUILD || '' }}"
+          PACKAGES: "${{ env.PACKAGES }}"
           INDEX: 1
           NO_DEFAULT_FEEDS: 1
           NO_REFRESH_CHECK: 1
           NO_SHFMT_CHECK: 1
 
+      - name: upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.arch }}-${{ matrix.version }}
+          path: bin/packages/${{ matrix.arch }}/libremesh/
+
+  publish:
+    name: publish ${{ github.ref }}
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    needs: build
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix: ${{ fromJson(needs.generate_matrix.outputs.matrix_builds) }}
+      # matrix:
+      #   version: ['main']
+      #   arch: ['x86_64']
+      #   output_path: ['openwrt_main']
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: "${{ matrix.arch }}-${{ matrix.version }}"
+          path: artifacts
+
       - name: Set package destination
         run: |
+          ls ./artifacts
           export TAG=$(echo "${{ github.ref }}" | cut -d '/' -f 3- | perl -pe 's/v([0-9])/$1/')
           echo "$TAG"
           echo "DEST_DIR=$TAG" >> $GITHUB_ENV
 
       - name: Upload packages to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
-          external_repository: libremesh/lime-feed
-          publish_dir: bin/packages/${{ matrix.arch }}/libremesh/
-          destination_dir: arch_packages/${{ env.DEST_DIR }}/${{ matrix.arch }}
+          # personal_token: ${{ secrets.PERSONAL_TOKEN }}
+          external_repository: a-gave/lime-feed
+          publish_dir: ./artifacts/
+          destination_dir: arch_packages/${{ env.DEST_DIR }}/${{ matrix.output_path }}/${{ matrix.arch }}/
 
diff --git a/README.md b/README.md
@@ -56,22 +56,41 @@ Start an ImageBuilder of your choice, for example ath79-generic if your device i
 
 ```shell
 mkdir ./images/
-docker run -it -v $(pwd)/images:/images/ ghcr.io/openwrt/imagebuilder:ath79-generic-v23.05.3
+docker run -it -v $(pwd)/images:/images/ ghcr.io/openwrt/imagebuilder:ath79-generic-v23.05.5
 ```
 
 If your device is not part of ath79-generic profiles, you can replace it with another &lt;target&gt;-&lt;subtarget&gt; combination. For knowing which target and subtarget is best suited for your router, check out the page about it in the [OpenWrt's Table of Hardware][OpenWrt-ToH].
 
 Within the container, add the `lime-packages` feeds:
 
+**opkg on openwrt 24.10.x and previous**
+
 ```shell
 echo "src/gz libremesh https://feed.libremesh.org/master" >> repositories.conf
 echo "src/gz libremesh_profiles https://feed.libremesh.org/profiles" >> repositories.conf
-echo "src/gz libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/mips_24kc" >> repositories.conf
 echo  "untrusted comment: signed by libremesh.org key a71b3c8285abd28b" > keys/a71b3c8285abd28b
 echo "RWSnGzyChavSiyQ+vLk3x7F0NqcLa4kKyXCdriThMhO78ldHgxGljM/8" >> keys/a71b3c8285abd28b
 ```
+To use the new `shared-state-async` add this repo:
+- replace `openwrt_23.05` with the openwrt branch 
+- replace `mips_24kc` with the architecture of your device based on target/subtarget
+
+```shell 
+echo "src/gz libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/openwrt_23.05/mips_24kc" >> repositories.conf
+```
 
-If your device is not part of ath79-generic replace `mips_24kc` with the architecture of the selected &lt;target&gt;/&lt;subtarget&gt;.
+**apk on openwrt main branch**
+```
+echo "@libremesh https://feed.libremesh.org/apk/master/packages.adb" >> repositories
+echo "@libremesh_profiles https://feed.libremesh.org/apk/profiles/packages.adb" >> repositories
+```
+To use the new `shared-state-async` add this repo: 
+- replace `openwrt_main` with the openwrt branch 
+- replace `mips_24kc` with the architecture of your device based on target/subtarget
+
+```shell 
+echo "@libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/openwrt_main/mips_24kc/packages.adb" >> repositories
+```
 
 Ideally add your own `lime-community` files within the container in the folder
 `./files/etc/config/`. To find possible options consult the
@@ -94,14 +113,9 @@ Your images should be available outside of the container in the `./images/` fold
 
 Go to <https://firmware-selector.openwrt.org/>. Find your device. Click on the folder symbol right after "Links: ". Alternatively, find your device in [OpenWrt's Table of Hardware][OpenWrt-ToH], find the image download link, remove the filename from the right side of the link and put the result in your browsers address bar. Scroll down and download openwrt-imagebuilder-*. Unpack the file and open a terminal inside the directory. Add the `lime-packages` feed:
 
-```shell
-echo "src/gz libremesh https://feed.libremesh.org/master" >> repositories.conf
-echo "src/gz libremesh_profiles https://feed.libremesh.org/profiles" >> repositories.conf
-echo "src/gz libremesh_arch_packages https://feed.libremesh.org/arch_packages/master/mips_24kc" >> repositories.conf
-echo  "untrusted comment: signed by libremesh.org key a71b3c8285abd28b" > keys/a71b3c8285abd28b
-echo "RWSnGzyChavSiyQ+vLk3x7F0NqcLa4kKyXCdriThMhO78ldHgxGljM/8" >> keys/a71b3c8285abd28b
-```
-If your device is not part of ath79-generic replace `mips_24kc` with the architecture of the selected &lt;target&gt;/&lt;subtarget&gt;.
+Follow the same steps at With Docker
+- **opkg on openwrt 24.10.x and previous**
+- **apk on openwrt main branch**
 
 Create an image with
 ```shell

diff --git a/packages/lime-system/files/etc/uci-defaults/92_add-lime-repos b/packages/lime-system/files/etc/uci-defaults/92_add-lime-repos
@@ -5,31 +5,57 @@
 	exit 0
 }
 
-feeds_file="/etc/opkg/limefeeds.conf"
+repo=$([ -f /etc/opkg/distfeed.conf ] && echo "opkg" \
+	|| ([ -f /etc/apk/repositories ] && echo "apk" ) ) || {
+		echo "Package manager not found - skipping"
+		exit 0
+	}
+
+feeds_file=$([ $repo == 'opkg' ] && echo "/etc/opkg/limefeeds.conf" \
+	|| ([ $repo == 'apk' ] && echo "/etc/apk/repositories") )
+
+$( ([ $repo == 'opkg' ] && [ -f "$feeds_file" ]) \
+  || ([ $repo == 'apk' ] && $(grep -q '@libremesh' $feeds_file)) ) && 
+{
+  echo "LibreMesh $repo feeds already defined - skipping"
+  exit 0;
+}
+
+arch="$(grep OPENWRT_ARCH /etc/os-release | sed 's/OPENWRT_ARCH=\"\(.*\)\"/\1/')"
 
-[ -f "$feeds_file" ] && {
-  echo "LibreMesh opkg feeds already defined - skipping"
-  exit 0
+openwrt_branch_ref="$(grep -m 1 "openwrt.org/" $feeds_file | sed 's|.*openwrt.org/\(.*\)|\1|' )"
+openwrt_branch=$(echo $openwrt_branch_ref | grep -q 'snapshots' && echo 'openwrt_main' )
+$(echo $openwrt_branch_ref | grep -q 'releases') && {
+	branch_n="$(echo $openwrt_branch_ref | sed 's/releases\///')"
+	openwrt_branch="openwrt_${branch_n:0:5}"
 }
 
-arch_packages="$(grep OPENWRT_ARCH /etc/os-release | sed 's/OPENWRT_ARCH=\"\(.*\)\"/\1/')"
+main_feed_path=$([ $repo == 'opkg' ] && echo "" \
+	|| ([ $repo == 'apk' ] && echo "apk/") )
 
 [ "$LIME_CODENAME" == "development" ] && {
-	packages_url="http://feed.libremesh.org/master";
-	arch_packages_url="http://feed.libremesh.org/arch_packages/master/$arch_packages";
+	packages_url="http://feed.libremesh.org/${main_feed_path}master";
+	arch_packages_url="http://feed.libremesh.org/arch_packages/master/$openwrt_branch/$arch/";
 } || {
-	packages_url="http://feed.libremesh.org/$LIME_RELEASE"
-	arch_packages_url="http://feed.libremesh.org/arch_packages/$LIME_RELEASE/$arch_packages";
+	packages_url="http://feed.libremesh.org/${main_feed_path}$LIME_RELEASE"
+	arch_packages_url="http://feed.libremesh.org/arch_packages/$LIME_RELEASE/$openwrt_branch/$arch/";
 }
 
 profiles_url="http://feed.libremesh.org/profiles"
 
 key_name="a71b3c8285abd28b"
 key_content="RWSnGzyChavSiyQ+vLk3x7F0NqcLa4kKyXCdriThMhO78ldHgxGljM/8"
 
-echo "Configuring official LibreMesh opkg feeds"
+echo "Configuring official LibreMesh $repo feeds"
+[ $repo == 'opkg' ] && {
 echo "src/gz libremesh $packages_url" > "$feeds_file"
 echo "src/gz libremesh_arch_packages $arch_packages_url" >> "$feeds_file"
 echo "src/gz profiles $profiles_url" >> "$feeds_file"
 echo "untrusted comment: signed by libremesh.org key $key_name" > "/etc/opkg/keys/$key_name"
 echo "$key_content" >> "/etc/opkg/keys/$key_name"
+}
+[ $repo == 'apk' ] && {
+echo "@libremesh $packages_url/packages.adb" >> "$feeds_file"
+echo "@libremesh_arch_packages $arch_packages_url/packages.adb" >> "$feeds_file"
+echo "@profiles $profiles_url/packages.adb" >> "$feeds_file"
+}