Merge pull request #435 from TheYoctoJester/kirkstone

adjust kas and Github Actions to scarthgap state
mendersoftware · Feb 5, 2025 · 2e9d5e8 · 2e9d5e8
2 parents 1068f34 + c328cee
commit 2e9d5e8
Show file tree

Hide file tree

Showing 12 changed files with 610 additions and 36 deletions.
diff --git a/.github/workflows/build.yml → .github/workflows/build_boards.yml b/.github/workflows/build.yml → .github/workflows/build_boards.yml
@@ -1,4 +1,4 @@
-name: build boards
+name: build_boards
 
 on:
   workflow_dispatch:
@@ -7,23 +7,23 @@ on:
     - cron:  '5 2 * * 6'
 
 env:
-  BUILDDIR: gh
+  BUILDDIR: build
 
 jobs:
   prepare:
-    runs-on: [self-hosted, linux, x64]
+    runs-on: [self-hosted, linux, x64, builder]
+    outputs:
+      artifact_path: ${{ steps.artifact_path.outputs.artifact_path }}
     steps:
       - name: Clean up
         run: rm -fR $BUILDDIR
       - uses: actions/checkout@v4
-      - name: cache metadata layers
-        run: ./ci/update-repos
-        env:
-          LAYERCACHE: ${{ vars.PERSISTENT_DIR }}/layers
+      - id: artifact_path
+        name: create artifact output directory
+        run: echo "artifact_path=${{ vars.ARTIFACT_OUTPUT_DIR }}/$GITHUB_REPOSITORY/$GITHUB_WORKFLOW/$GITHUB_RUN_NUMBER-$GITHUB_RUN_ATTEMPT" >> $GITHUB_OUTPUT
 
   build:
     needs: [prepare]
-    continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
@@ -50,11 +50,6 @@ jobs:
           jetson-xavier-nx-devkit,
           libretech-ac,
           libretech-cc,
-          nitrogen8m,
-          nitrogen8mm,
-          nitrogen8mn,
-          nitrogen8mp,
-          osd32mp1-emmc-mender,
           raspberrypi,
           raspberrypi-cm,
           raspberrypi0,
@@ -71,52 +66,49 @@ jobs:
           qemux86-64,
           vexpress-qemu,
           vexpress-qemu-flash,
-          colibri-imx8x,
-          verdin-imx8mm,
-          verdin-imx8mp,
-          imx8mm-var-dart,
-          sama5d27-som1-ek-sd,
-          sama5d3-xplained,
           intel-corei7-64,
           intel-corei7-64-sdcard,
           intel-nuc-ssd,
           cubox-i,
           rock-pi-e,
-          apalis-imx8, 
           apalis-imx8-boot2qt,
           x86-virtual,
           jetson-orin-nano-devkit,
           jetson-orin-nano-devkit-nvme,
           jetson-agx-orin-devkit,
         ]
-        experimental: [false]
         subpath: [.]
-        include:
-          - board: apalis-imx6
-            experimental: true
-          - board: colibri-imx6ull
-            experimental: true
-          - board: stm32mp15-disco
-            experimental: true
-          - board: qemuarm64-swupdate
-            subpath: demos
-            experimental: true
-          - board: jetson-agx-orin-devkit
-            experimental: true
 
-    runs-on: [self-hosted, linux, x64]
+    runs-on: [self-hosted, linux, x64, builder]
     container:
       # the container image needs to be effectively hardcoded, it seems.
       image: ghcr.io/theyoctojester/devcontainer-base-yoep:main
       volumes:
         - ${{ vars.PERSISTENT_DIR }}/downloads:${{ vars.BUILD_DL_DIR }}
         - ${{ vars.PERSISTENT_DIR }}/sstate-cache:${{ vars.BUILD_SSTATE_DIR }}
         - ${{ vars.PERSISTENT_DIR }}/layers:${{ vars.BUILD_LAYERCACHE_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/assets:${{ vars.BUILD_ASSETS_DIR }}
       options: --user ${{ vars.KAS_UID }}:${{ vars.KAS_GID }}
     steps:
       - name: enter build dir and build
         env:
           DL_DIR: ${{ vars.BUILD_DL_DIR}}
           SSTATE_DIR: ${{ vars.BUILD_SSTATE_DIR}}
           KAS_REPO_REF_DIR: ${{ vars.BUILD_LAYERCACHE_DIR}}
-        run: mkdir -p $BUILDDIR/${{ matrix.board }} && cd $BUILDDIR/${{ matrix.board }} && kas build ../../kas/${{ matrix.subpath }}/${{ matrix.board }}.yml
+        run: >
+          mkdir -p $BUILDDIR/${{ matrix.board }} &&
+          cd $BUILDDIR/${{ matrix.board }} &&
+          kas checkout ../../kas/${{ matrix.subpath }}/${{ matrix.board }}.yml &&
+          if [ -f "${{ vars.BUILD_ASSETS_DIR }}/site.conf" ]; then 
+          cp ${{ vars.BUILD_ASSETS_DIR }}/site.conf build/conf/site.conf;
+          fi &&
+          kas build ../../kas/${{ matrix.subpath }}/${{ matrix.board }}.yml
+
+  collect:
+    needs: [prepare, build]
+    runs-on: [self-hosted, linux, x64, builder]
+    steps:
+      - name: collect artifacts
+        env:
+          ARTIFACT_PATH: ${{ needs.prepare.outputs.artifact_path }}
+        run: ./ci/collect_artifacts.sh
diff --git a/.github/workflows/build_demos.yml b/.github/workflows/build_demos.yml
@@ -0,0 +1,67 @@
+name: build_demos
+
+on:
+  workflow_dispatch:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron:  '5 2 * * 6'
+
+env:
+  BUILDDIR: demos
+
+jobs:
+  prepare:
+    runs-on: [self-hosted, linux, x64, builder]
+    outputs:
+      artifact_path: ${{ steps.artifact_path.outputs.artifact_path }}
+    steps:
+      - name: Clean up
+        run: rm -fR $BUILDDIR
+      - uses: actions/checkout@v4
+      - id: artifact_path
+        name: create artifact output directory
+        run: echo "artifact_path=${{ vars.ARTIFACT_OUTPUT_DIR }}/$GITHUB_REPOSITORY/$GITHUB_WORKFLOW/$GITHUB_RUN_NUMBER-$GITHUB_RUN_ATTEMPT" >> $GITHUB_OUTPUT
+
+  build:
+    needs: [prepare]
+    strategy:
+      fail-fast: false
+      matrix:
+        board: [
+          qemuarm64-swupdate
+        ]
+        subpath: [demos]
+
+    runs-on: [self-hosted, linux, x64, builder]
+    container:
+      # the container image needs to be effectively hardcoded, it seems.
+      image: ghcr.io/theyoctojester/devcontainer-base-yoep:main
+      volumes:
+        - ${{ vars.PERSISTENT_DIR }}/downloads:${{ vars.BUILD_DL_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/sstate-cache:${{ vars.BUILD_SSTATE_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/layers:${{ vars.BUILD_LAYERCACHE_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/assets:${{ vars.BUILD_ASSETS_DIR }}
+      options: --user ${{ vars.KAS_UID }}:${{ vars.KAS_GID }}
+    steps:
+      - name: enter build dir and build
+        env:
+          DL_DIR: ${{ vars.BUILD_DL_DIR}}
+          SSTATE_DIR: ${{ vars.BUILD_SSTATE_DIR}}
+          KAS_REPO_REF_DIR: ${{ vars.BUILD_LAYERCACHE_DIR}}
+        run: >
+          mkdir -p $BUILDDIR/${{ matrix.board }} &&
+          cd $BUILDDIR/${{ matrix.board }} &&
+          kas checkout ../../kas/${{ matrix.subpath }}/${{ matrix.board }}.yml &&
+          if [ -f "${{ vars.BUILD_ASSETS_DIR }}/site.conf" ]; then 
+          cp ${{ vars.BUILD_ASSETS_DIR }}/site.conf build/conf/site.conf;
+          fi &&
+          kas build ../../kas/${{ matrix.subpath }}/${{ matrix.board }}.yml
+
+  collect:
+    needs: [prepare, build]
+    runs-on: [self-hosted, linux, x64, builder]
+    steps:
+      - name: collect artifacts
+        env:
+          ARTIFACT_PATH: ${{ needs.prepare.outputs.artifact_path }}
+        run: ./ci/collect_artifacts.sh
diff --git a/.github/workflows/build_experimental.yml b/.github/workflows/build_experimental.yml
@@ -0,0 +1,69 @@
+name: build_experimental
+
+on:
+  workflow_dispatch:
+  workflow_run:
+    workflows: ["cache layers"]
+    types: 
+    - completed
+
+env:
+  BUILDDIR: experimental
+
+jobs:
+  prepare:
+    runs-on: [self-hosted, linux, x64, builder]
+    steps:
+      - name: Clean up
+        run: rm -fR $BUILDDIR
+      - uses: actions/checkout@v4
+
+  build:
+    needs: [prepare]
+    strategy:
+      fail-fast: false
+      matrix:
+        board: [
+          apalis-imx6,
+          apalis-imx8, 
+          colibri-imx6ull,
+          colibri-imx8x,
+          imx8mm-var-dart,
+          jetson-agx-orin-devkit,
+          nitrogen8m,
+          nitrogen8mm,
+          nitrogen8mn,
+          nitrogen8mp,
+          osd32mp1-emmc-mender,
+          sama5d27-som1-ek-sd,
+          sama5d3-xplained,
+          stm32mp15-disco,
+          verdin-imx8mm,
+          verdin-imx8mp,
+        ]
+        subpath: [.]
+
+    runs-on: [self-hosted, linux, x64, builder]
+    container:
+      # the container image needs to be effectively hardcoded, it seems.
+      image: ghcr.io/theyoctojester/devcontainer-base-yoep:main
+      volumes:
+        - ${{ vars.PERSISTENT_DIR }}/downloads:${{ vars.BUILD_DL_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/sstate-cache:${{ vars.BUILD_SSTATE_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/layers:${{ vars.BUILD_LAYERCACHE_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/assets:${{ vars.BUILD_ASSETS_DIR }}
+      options: --user ${{ vars.KAS_UID }}:${{ vars.KAS_GID }}
+    steps:
+      - name: enter build dir and build
+        env:
+          DL_DIR: ${{ vars.BUILD_DL_DIR}}
+          SSTATE_DIR: ${{ vars.BUILD_SSTATE_DIR}}
+          KAS_REPO_REF_DIR: ${{ vars.BUILD_LAYERCACHE_DIR}}
+        run: >
+          mkdir -p $BUILDDIR/${{ matrix.board }} &&
+          cd $BUILDDIR/${{ matrix.board }} &&
+          kas checkout ../../kas/${{ matrix.subpath }}/${{ matrix.board }}.yml &&
+          if [ -f "${{ vars.BUILD_ASSETS_DIR }}/site.conf" ]; then 
+          cp ${{ vars.BUILD_ASSETS_DIR }}/site.conf build/conf/site.conf;
+          fi &&
+          kas build ../../kas/${{ matrix.subpath }}/${{ matrix.board }}.yml
diff --git a/.github/workflows/build_validation.yml b/.github/workflows/build_validation.yml
@@ -0,0 +1,82 @@
+name: build_validation
+
+on:
+  workflow_dispatch:
+  push:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron:  '5 2 * * 6'
+
+env:
+  BUILDDIR: validation
+
+jobs:
+  prepare:
+    runs-on: [self-hosted, linux, x64, builder]
+    outputs:
+      artifact_path: ${{ steps.artifact_path.outputs.artifact_path }}
+      artifact_latest: ${{ steps.artifact_latest.outputs.artifact_latest }}
+    steps:
+      - name: Clean up
+        run: rm -fR $BUILDDIR
+      - uses: actions/checkout@v4
+      - id: artifact_path
+        name: generate artifact output directory name
+        run: echo "artifact_path=${{ vars.ARTIFACT_OUTPUT_DIR }}/$GITHUB_REPOSITORY/$GITHUB_REF_NAME/$GITHUB_WORKFLOW/$GITHUB_RUN_NUMBER-$GITHUB_RUN_ATTEMPT" >> $GITHUB_OUTPUT
+      - id: artifact_latest
+        name: generate latest artifact symlink name
+        run: echo "artifact_latest=${{ vars.ARTIFACT_OUTPUT_DIR }}/$GITHUB_REPOSITORY/$GITHUB_REF_NAME/$GITHUB_WORKFLOW/latest" >> $GITHUB_OUTPUT
+
+  build:
+    needs: [prepare]
+    strategy:
+      fail-fast: false
+      matrix:
+        board: [
+          beaglebone,
+          raspberrypi3,
+          raspberrypi3-64,
+          raspberrypi4,
+          raspberrypi4-64,
+          raspberrypi5,
+        ]
+        subpath: [.]
+
+    runs-on: [self-hosted, linux, x64, builder]
+    container:
+      # the container image needs to be effectively hardcoded, it seems.
+      image: ghcr.io/theyoctojester/devcontainer-base-yoep:main
+      volumes:
+        - ${{ vars.PERSISTENT_DIR }}/downloads:${{ vars.BUILD_DL_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/sstate-cache:${{ vars.BUILD_SSTATE_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/layers:${{ vars.BUILD_LAYERCACHE_DIR }}
+        - ${{ vars.PERSISTENT_DIR }}/assets:${{ vars.BUILD_ASSETS_DIR }}
+      options: --user ${{ vars.KAS_UID }}:${{ vars.KAS_GID }}
+    steps:
+      - name: enter build dir and build
+        env:
+          DL_DIR: ${{ vars.BUILD_DL_DIR}}
+          SSTATE_DIR: ${{ vars.BUILD_SSTATE_DIR}}
+          KAS_REPO_REF_DIR: ${{ vars.BUILD_LAYERCACHE_DIR}}
+        run: >
+          mkdir -p $BUILDDIR/${{ matrix.board }} &&
+          cd $BUILDDIR/${{ matrix.board }} &&
+          kas checkout ../../kas/${{ matrix.subpath }}/${{ matrix.board }}.yml &&
+          if [ -f "${{ vars.BUILD_ASSETS_DIR }}/site.conf" ]; then 
+          cp ${{ vars.BUILD_ASSETS_DIR }}/site.conf build/conf/site.conf;
+          fi &&
+          kas build ../../kas/${{ matrix.subpath }}/${{ matrix.board }}.yml:../../kas/include/validation.yml
+
+  collect:
+    needs: [prepare, build]
+    runs-on: [self-hosted, linux, x64, builder]
+    steps:
+      - name: collect artifacts
+        env:
+          ARTIFACT_PATH: ${{ needs.prepare.outputs.artifact_path }}
+        run: ./ci/collect_artifacts.sh
+      - name: mark latest
+        run: >
+          rm -fR ${{ needs.prepare.outputs.artifact_latest }};
+          echo "using artifact path ${{ needs.prepare.outputs.artifact_path }}, latest ${{ needs.prepare.outputs.artifact_latest }}" &&
+          ln -s ${{ needs.prepare.outputs.artifact_path }} ${{ needs.prepare.outputs.artifact_latest }}
diff --git a/.github/workflows/cache_layers.yml b/.github/workflows/cache_layers.yml
@@ -0,0 +1,17 @@
+name: cache_layers
+
+on:
+  workflow_dispatch:
+  push:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron:  '5 1 * * *'
+
+jobs:
+  prepare:
+    runs-on: [self-hosted, linux, x64, builder]
+    steps:
+      - name: cache metadata layers
+        run: ./ci/update-repos
+        env:
+          LAYERCACHE: ${{ vars.PERSISTENT_DIR }}/layers