re-enable schedules, bump to 2/2 chunks, drop old workflow

armsurvivors · Mar 8, 2024 · fdcefe4 · fdcefe4
1 parent 950e7c4
commit fdcefe4
Show file tree

Hide file tree

Showing 3 changed files with 194 additions and 54 deletions.
diff --git a/.github/workflows/artifact-image-complete-matrix.yml b/.github/workflows/artifact-image-complete-matrix.yml
@@ -5,8 +5,8 @@
 name: "artifact-image-complete-matrix"
 
 on:
-  #schedule:
-  #  - cron: '0 1 * * *' # Scheduled runs every day at 2am UTC
+  schedule:
+    - cron: '0 2 * * *' # Scheduled runs every day at 2am UTC
   workflow_dispatch:
     inputs:
       skipImages:
@@ -53,7 +53,7 @@ env:
 
   # Armbian envs. Adjust to your needs.
   # OCI_TARGET_BASE: "ghcr.io/${{ github.repository }}/" # This is picked up by the Docker launcher automatically. It does _not_ override the targets.yaml during info resolution.
-  DOCKER_ARMBIAN_BASE_COORDINATE_PREFIX: "ghcr.io/${{ github.repository }}:armbian-next-" # Use Docker image in same repo
+  DOCKER_ARMBIAN_BASE_COORDINATE_PREFIX: "ghcr.io/armsurvivors/armbian-builder:armbian-next-" # Use Docker image in same repo
   DOCKER_SKIP_UPDATE: "yes" # Do not apt update/install/requirements/etc during Dockerfile build, trust DOCKER_ARMBIAN_BASE_COORDINATE_PREFIX's images are up-to-date
 
   # Added to every build, even the prepare job.
@@ -65,7 +65,7 @@ env:
 jobs:
 
   matrix_prep:
-    name: "JSON matrix: 1/1 :: 1 artifact chunks, 1 image chunks"
+    name: "JSON matrix: 2/2 :: 2 artifact chunks, 2 image chunks"
     if: ${{ github.repository_owner == 'armsurvivors' }}
     runs-on: [ "self-hosted", "Linux", 'armbian', "matrix-prepare" ]
     steps:
@@ -141,8 +141,8 @@ jobs:
           bash ./compile.sh gha-matrix rpardini-generic ${{env.EXTRA_PARAMS_ALL_BUILDS}} \
             REVISION="${{ steps.prepare-date-outputs.outputs.revision_release }}" \
             TARGETS_FILTER_INCLUDE='${{ github.event.inputs.targetsFilterInclude || '' }}' \
-            MATRIX_ARTIFACT_CHUNKS=1 \
-            MATRIX_IMAGE_CHUNKS=1 \
+            MATRIX_ARTIFACT_CHUNKS=2 \
+            MATRIX_IMAGE_CHUNKS=2 \
             BETA=${{ github.event.inputs.nightly || 'yes' }} \
             CHECK_OCI=${{ github.event.inputs.checkOci || 'yes' }} \
             TARGETS_FILENAME=targets.yaml \
@@ -180,42 +180,50 @@ jobs:
 
 # template file: 150.per-chunk-artifacts_prep-outputs.yaml
 
-      # artifacts-1 of 1
+      # artifacts-1 of 2
       artifacts-chunk-json-1: ${{ steps.prepare-matrix.outputs.artifacts-chunk-json-1 }}
       artifacts-chunk-not-empty-1: ${{ steps.prepare-matrix.outputs.artifacts-chunk-not-empty-1 }}
       artifacts-chunk-size-1: ${{ steps.prepare-matrix.outputs.artifacts-chunk-size-1 }}
+      # artifacts-2 of 2
+      artifacts-chunk-json-2: ${{ steps.prepare-matrix.outputs.artifacts-chunk-json-2 }}
+      artifacts-chunk-not-empty-2: ${{ steps.prepare-matrix.outputs.artifacts-chunk-not-empty-2 }}
+      artifacts-chunk-size-2: ${{ steps.prepare-matrix.outputs.artifacts-chunk-size-2 }}
 
 # template file: 151.per-chunk-images_prep-outputs.yaml
 
-      # artifacts-1 of 1
+      # artifacts-1 of 2
       images-chunk-json-1: ${{ steps.prepare-matrix.outputs.images-chunk-json-1 }}
       images-chunk-not-empty-1: ${{ steps.prepare-matrix.outputs.images-chunk-not-empty-1 }}
       images-chunk-size-1: ${{ steps.prepare-matrix.outputs.images-chunk-size-1 }}
+      # artifacts-2 of 2
+      images-chunk-json-2: ${{ steps.prepare-matrix.outputs.images-chunk-json-2 }}
+      images-chunk-not-empty-2: ${{ steps.prepare-matrix.outputs.images-chunk-not-empty-2 }}
+      images-chunk-size-2: ${{ steps.prepare-matrix.outputs.images-chunk-size-2 }}
 
 # template file: 250.single_aggr-jobs.yaml
 
   # ------ aggregate all artifact chunks into a single dependency -------
 
   all-artifacts-ready:
-    name: "1 artifacts chunks ready"
+    name: "2 artifacts chunks ready"
     runs-on: ubuntu-latest # not going to run, anyway, but is required.
     if: ${{ !cancelled() && ( 1 == 2 ) }} # eg: never run.
-    needs: [ "matrix_prep", "build-artifacts-chunk-1" ] # <-- HERE: all artifact chunk numbers.
+    needs: [ "matrix_prep", "build-artifacts-chunk-1","build-artifacts-chunk-2" ] # <-- HERE: all artifact chunk numbers.
     steps:
       - name: fake step
         run: uptime
 
   all-images-ready:
-    name: "1 image chunks ready"
+    name: "2 image chunks ready"
     runs-on: ubuntu-latest # not going to run, anyway, but is required.
     if: ${{ !cancelled() && ( 1 == 2 ) }} # eg: never run.
-    needs: [ "matrix_prep", "build-images-chunk-1" ] # <-- HERE: all image chunk numbers.
+    needs: [ "matrix_prep", "build-images-chunk-1","build-images-chunk-2" ] # <-- HERE: all image chunk numbers.
     steps:
       - name: fake step
         run: uptime
 
   all-artifacts-and-images-ready:
-    name: "1 artifact chunks and 1 image chunks ready"
+    name: "2 artifact chunks and 2 image chunks ready"
     runs-on: ubuntu-latest # not going to run, anyway, but is required.
     if: ${{ !cancelled() && ( 1 == 2 ) }} # eg: never run.
     needs: [ "matrix_prep", "all-artifacts-ready", "all-images-ready" ]
@@ -224,7 +232,7 @@ jobs:
         run: uptime
 
   all-artifacts-and-repo-ready:
-    name: "1 artifact chunks and apt repository ready"
+    name: "2 artifact chunks and apt repository ready"
     runs-on: ubuntu-latest # not going to run, anyway, but is required.
     if: ${{ !cancelled() && ( 1 == 2 ) }} # eg: never run.
     needs: [ "matrix_prep", "all-artifacts-ready", "publish-debs-to-repo" ]
@@ -297,6 +305,69 @@ jobs:
         run: |
           echo "Logs: ${{ steps.build.outputs.logs_url }}"
 
+  "build-artifacts-chunk-2": # templated "build-artifacts-chunk-2"
+    if: ${{ github.repository_owner == 'armsurvivors' && needs.matrix_prep.outputs.artifacts-chunk-not-empty-2 == 'yes' }}  # <-- HERE: Chunk number.
+    needs: [ "matrix_prep" ]
+    strategy:
+      fail-fast: false # let other jobs try to complete if one fails
+      matrix: ${{ fromJSON(needs.matrix_prep.outputs.artifacts-chunk-json-2) }} # <-- HERE: Chunk number.
+    name: ${{ matrix.desc || 'Empty A2' }} # <-- HERE: Chunk number.
+    timeout-minutes: 120
+    runs-on: ${{ matrix.runs_on }}
+    steps:
+
+      #- name: Runner clean
+      #  uses: armbian/actions/runner-clean@main
+
+      # Login to ghcr.io, for later uploading rootfs to ghcr.io
+      - name: Docker Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: "${{ github.repository_owner }}" # GitHub username or org
+          password: ${{ secrets.GITHUB_TOKEN }}    # GitHub actions builtin token. repo has to have pkg access.
+
+      # cleanup the place where we will clone the userpatches repo, to avoid git going insane and cleaning everything later
+      - name: Cleanup userpatches repo
+        if: ${{ ( env.USERPATCHES_REPOSITORY != '' ) && ( env.USERPATCHES_REF != '' ) }}
+        run: rm -rf userpatches.repo
+
+      - name: "Checkout build repo with depth ${{ matrix.fdepth }}"
+        uses: actions/checkout@v3 # We don't need to clone git, really. A wget would suffice for GH-hosted runners. But using clone is better for Igor-hosted runners.
+        with:
+          repository: ${{ env.BUILD_REPOSITORY }}
+          ref: ${{ needs.matrix_prep.outputs.build-sha1 }}
+          fetch-depth: ${{ matrix.fdepth }} # fetch all history for self-hosted, but shallow for GH-hosted
+          clean: false # true is default. it *will* delete the hosts /dev if mounted inside.
+
+      # clone the userpatches repo (`armbian/os`)
+      - name: "Checkout userpatches repo with depth ${{ matrix.fdepth }}: ${{env.USERPATCHES_REPOSITORY}}#${{env.USERPATCHES_REF}}"
+        uses: actions/checkout@v3
+        if: ${{ ( env.USERPATCHES_REPOSITORY != '' ) && ( env.USERPATCHES_REF != '' ) }}
+        with:
+          repository: ${{ env.USERPATCHES_REPOSITORY }}
+          ref: ${{ env.USERPATCHES_REF }}
+          fetch-depth: ${{ matrix.fdepth }} # fetch all history for self-hosted, but shallow for GH-hosted
+          clean: false # true is default.
+          path: userpatches.repo
+
+      - name: "Put userpatches in place, and remove userpatches repo"
+        if: ${{ ( env.USERPATCHES_REPOSITORY != '' ) && ( env.USERPATCHES_REF != '' ) }}
+        run: |
+          mkdir -pv userpatches
+          rsync -av userpatches.repo/${{env.USERPATCHES_DIR}}/. userpatches/
+          rm -rf userpatches.repo
+
+      - name: Build  ${{matrix.desc}}
+        id: build
+        run: |
+          bash ./compile.sh ${{ matrix.invocation }} ${{env.EXTRA_PARAMS_ALL_BUILDS}} UPLOAD_TO_OCI_ONLY=yes
+
+      - name: "Logs: ${{ steps.build.outputs.logs_url }}"
+        if: always()
+        run: |
+          echo "Logs: ${{ steps.build.outputs.logs_url }}"
+
 
 # template file: 650.per-chunk-images_job.yaml
 
@@ -387,6 +458,93 @@ jobs:
         if: always()
         run: |
           rm -rfv output/images userpatches/VERSION
+  "build-images-chunk-2": # templated "build-images-chunk-2"
+    needs: [ "matrix_prep", "all-artifacts-and-repo-ready" ]
+    timeout-minutes: 60
+    if: ${{ !failure() && !cancelled() && ( github.repository_owner == 'armsurvivors' ) && ( needs.matrix_prep.outputs.images-chunk-not-empty-2 == 'yes' ) }} # <-- HERE: Chunk number.
+    strategy:
+      fail-fast: false # let other jobs try to complete if one fails
+      matrix: ${{ fromJSON(needs.matrix_prep.outputs.images-chunk-json-2) }} # <-- HERE: Chunk number.
+    name: ${{ matrix.desc || 'Empty I2' }} # <-- HERE: Chunk number.
+    runs-on: ${{ matrix.runs_on }}
+    steps:
+
+      # Login to ghcr.io, we'll be downloading a lot from ghcr.io
+      - name: Docker Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: "${{ github.repository_owner }}" # GitHub username or org
+          password: ${{ secrets.GITHUB_TOKEN }}    # GitHub actions builtin token. repo has to have pkg access.
+
+      # cleanup the place where we will clone the userpatches repo, to avoid git going insane and cleaning everything later
+      - name: Cleanup userpatches repo
+        if: ${{ ( env.USERPATCHES_REPOSITORY != '' ) && ( env.USERPATCHES_REF != '' ) }}
+        run: rm -rf userpatches.repo
+
+      - name: "Checkout build repo with depth ${{ matrix.fdepth }}"
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ env.BUILD_REPOSITORY }}
+          ref: ${{ needs.matrix_prep.outputs.build-sha1 }}
+          clean: false # true is default. it *will* delete the hosts /dev if mounted inside.
+          fetch-depth: ${{ matrix.fdepth }} # fetch all history for self-hosted, but shallow for GH-hosted
+
+      # clone the userpatches repo (`armbian/os`)
+      - name: "Checkout userpatches repo with depth ${{ matrix.fdepth }}: ${{env.USERPATCHES_REPOSITORY}}#${{env.USERPATCHES_REF}}"
+        uses: actions/checkout@v3
+        if: ${{ ( env.USERPATCHES_REPOSITORY != '' ) && ( env.USERPATCHES_REF != '' ) }}
+        with:
+          repository: ${{ env.USERPATCHES_REPOSITORY }}
+          ref: ${{ env.USERPATCHES_REF }}
+          fetch-depth: ${{ matrix.fdepth }} # fetch all history for self-hosted, but shallow for GH-hosted
+          clean: false # true is default.
+          path: userpatches.repo
+
+      - name: "Put userpatches in place, and remove userpatches repo"
+        if: ${{ ( env.USERPATCHES_REPOSITORY != '' ) && ( env.USERPATCHES_REF != '' ) }}
+        run: |
+          mkdir -pv userpatches
+          rsync -av userpatches.repo/${{env.USERPATCHES_DIR}}/. userpatches/
+          rm -rf userpatches.repo
+
+      - name: Cleanup leftover output images
+        run: |
+          rm -rfv output/images userpatches/VERSION
+
+      - name: ${{matrix.desc}}
+        id: build-one-image
+        run: |
+          bash ./compile.sh ${{ matrix.invocation }} ${{env.EXTRA_PARAMS_IMAGE}} ${{env.EXTRA_PARAMS_ALL_BUILDS}} OUTPUT_IMAGES_ONLY=yes
+
+      - name: "Logs: ${{ steps.build-one-image.outputs.logs_url }}"
+        if: always()
+        run: |
+          echo "Logs: ${{ steps.build-one-image.outputs.logs_url }}"
+
+      # @TODO lets upload to GHR release ourselves instead of this shit
+      - name: Release ${{ matrix.board }}
+        id: release1
+        continue-on-error: true
+        uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # de2c0eb8 = v0.1.5; already with koplo199's node16 fixes
+        with:
+          tag_name: ${{ needs.matrix_prep.outputs.images_release }}
+          files: |
+            output/images/*.xz
+
+      - name: Release ${{ matrix.board }} (retry if 1st failed)
+        id: release2
+        if: ${{ steps.release1.outcome == 'failure'}}
+        uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # de2c0eb8 = v0.1.5; already with koplo199's node16 fixes
+        with:
+          tag_name: ${{ needs.matrix_prep.outputs.images_release }}
+          files: |
+            output/images/*.xz
+
+      - name: Cleanup output images
+        if: always()
+        run: |
+          rm -rfv output/images userpatches/VERSION
 
 # template file: 750.single_repo.yaml
 

diff --git a/.github/workflows/daily-release.yml b/.github/workflows/daily-release.yml
diff --git a/.github/workflows/docker-image-simple.yml b/.github/workflows/docker-image-simple.yml
@@ -1,8 +1,8 @@
 name: docker-image-simple
 
 on:
-  #schedule:
-  #  - cron: '30 0 * * *' # Scheduled runs every day at 0:30am UTC
+  schedule:
+    - cron: '30 0 * * *' # Scheduled runs every day at 0:30am UTC
   workflow_dispatch:
   workflow_call:
 
@@ -19,26 +19,26 @@ jobs:
             platforms: "linux/amd64,linux/arm64" # Ubuntu doesn't have yet: ,linux/riscv64
             cache-from: "type=gha" # all-automatic Github Actions caching
             cache-to: "type=gha,mode=max" # all-automatic Github Actions caching, max mode
-          ## WAIT ## - os: "debian"
-          ## WAIT ##   release: "bookworm"
-          ## WAIT ##   platforms: "linux/amd64,linux/arm64"
-          ## WAIT ##   cache-from: ""
-          ## WAIT ##   cache-to: ""
-          ## WAIT ## - os: "debian"
-          ## WAIT ##   release: "trixie"
-          ## WAIT ##   platforms: "linux/amd64,linux/arm64"
-          ## WAIT ##   cache-from: ""
-          ## WAIT ##   cache-to: ""
-          ## WAIT ## - os: "debian"
-          ## WAIT ##   release: "sid"
-          ## WAIT ##   platforms: "linux/amd64,linux/arm64" # debian:sid is the only image with riscv64 as of 2024-03-03, but is hasn't all the toolchains yet: ,linux/riscv64
-          ## WAIT ##   cache-from: ""
-          ## WAIT ##   cache-to: ""
-          ## WAIT ## - os: "ubuntu"
-          ## WAIT ##   release: "noble"
-          ## WAIT ##   platforms: "linux/amd64,linux/arm64" # Ubuntu doesn't have yet: ,linux/riscv64
-          ## WAIT ##   cache-from: ""
-          ## WAIT ##   cache-to: ""
+          - os: "debian"
+            release: "bookworm"
+            platforms: "linux/amd64,linux/arm64"
+            cache-from: ""
+            cache-to: ""
+          - os: "debian"
+            release: "trixie"
+            platforms: "linux/amd64,linux/arm64"
+            cache-from: ""
+            cache-to: ""
+          - os: "debian"
+            release: "sid"
+            platforms: "linux/amd64,linux/arm64" # debian:sid is the only image with riscv64 as of 2024-03-03, but is hasn't all the toolchains yet: ,linux/riscv64
+            cache-from: ""
+            cache-to: ""
+          - os: "ubuntu"
+            release: "noble"
+            platforms: "linux/amd64,linux/arm64" # Ubuntu doesn't have yet: ,linux/riscv64
+            cache-from: ""
+            cache-to: ""
     name: "${{ matrix.release }} (${{ matrix.os }})"
     env:
       DOCKERFILE_OS: "${{ matrix.os }}"