Merge pull request #3217 from oxen-io/ci/plop

chore: improve CI workflow speed and usage

.github/workflows/build-binaries.yml

@@ -1,3 +1,7 @@
+# There is a fair bit of duplication here, but it is the best to save our github free minutes for now.
+# We could save and restore cache to different jobs but that takes roughly 3 minutes to save,
+# so better run them in parrallel instead.
+
 name: Session Desktop
 
 on:
@@ -18,55 +22,141 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  # we want to publish on "push to master" only. When we don't want to publish, we want to upload artefacts
+  SHOULD_PUBLISH: ${{ github.event_name == 'push' && github.ref == 'master' }}
+
 jobs:
-  build:
-    runs-on: ${{ matrix.os }}
+  build_linux:
+    runs-on: ubuntu-20.04
     strategy:
-      fail-fast: false
       matrix:
-        # We want a mac arm64 build, and according to this https://github.com/actions/runner-images#available-images macos-14 is always arm64
-        # macos-14 is disabled for now as we hit our free tier limit for macos builds
-        os: [windows-2022, ubuntu-20.04, macos-12]
+        # this needs to be a valid target of https://www.electron.build/linux#target
+        pkg_to_build: ['deb', 'rpm', 'freebsd', 'AppImage']
+
     env:
-      SIGNAL_ENV: production
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - run: git config --global core.autocrlf false
 
       - name: Checkout git repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup & Build
         uses: ./actions/setup_and_build
+        with:
+          cache_suffix: ${{ matrix.pkg_to_build }}
 
       - name: Lint Files
-        # no need to lint files on all platforms. Just do it once on the quicker one
-        if: runner.os == 'Linux'
+        # no need to lint files on all platforms
         run: yarn lint-full
 
       - name: Enforce yarn.lock has no duplicates
+        # no need to dedup yarn.lock on all platforms
         uses: ./actions/deduplicate_fail
+
+        # we want to test on all platforms since some are testing the rendered menus (and are dependent on the platform)
+      - name: Unit Test
+        run: yarn test
+
+      - name: Build but do not publish ${{ matrix.pkg_to_build }}
+        # we want this to run always, except on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'false' }}
+        run: |
+          sed -i 's/"target": \["deb"\]/"target": "${{ matrix.pkg_to_build }}"/g' package.json && yarn build-release
+
+      - name: Upload artefacts ${{ matrix.pkg_to_build }}
+        # we want this to run always, except on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'false' }}
+        uses: ./actions/upload_prod_artefacts
+        with:
+          upload_prefix: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.pkg_to_build }}
+
+      - name: Build & publish ${{ matrix.pkg_to_build }}
+        # we want this to run only when on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'true' }}
+        run: |
+          sed -i 's/\"target\": \\[\"deb\"\\]/\"target\": \"${{ matrix.pkg_to_build }}\"/g' package.json; yarn build-release-publish
+
+  build_windows:
+    runs-on: windows-2022
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - run: git config --global core.autocrlf false
+
+      - name: Checkout git repo
+        uses: actions/checkout@v4
+
+      - name: Setup & Build
+        uses: ./actions/setup_and_build
+        with:
+          cache_suffix: 'windows_x64'
+
+        # we want to test on all platforms since some are testing the rendered menus (and are dependent on the platform)
       - name: Unit Test
         run: yarn test
 
       - name: Build but do not publish
         # we want this to run always, except on "push" to "master"
-        if: github.event_name != 'push' || github.ref != 'master'
-        uses: ./actions/build_no_publish
+        if: ${{ env.SHOULD_PUBLISH == 'false' }}
+        run: yarn build-release
+
+      - name: Upload artefacts
+        # we want this to run always, except on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'false' }}
+        uses: ./actions/upload_prod_artefacts
         with:
-          MAC_CERTIFICATE: ${{ secrets.MAC_CERTIFICATE }}
-          MAC_CERTIFICATE_PASSWORD: ${{ secrets.MAC_CERTIFICATE_PASSWORD }}
-          SIGNING_APPLE_ID: ${{ secrets.SIGNING_APPLE_ID }}
-          SIGNING_APP_PASSWORD: ${{ secrets.SIGNING_APP_PASSWORD }}
-          SIGNING_TEAM_ID: ${{ secrets.SIGNING_TEAM_ID }}
+          upload_prefix: ${{ runner.os }}-${{ runner.arch }}
 
       - name: Build & publish
         # we want this to run only when on "push" to "master"
-        if: github.event_name == 'push' && github.ref == 'master'
-        uses: ./actions/build_publish
+        if: ${{ env.SHOULD_PUBLISH == 'true' }}
+        run: yarn build-release-publish # No other args needed for windows publish
+
+  # We want a mac arm64 build, and according to this https://github.com/actions/runner-images#available-images macos-14 is always arm64
+  # macos-14 is disabled for now as we hit our free tier limit for macos builds
+  build_macos_x64:
+    runs-on: macos-12
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      MAC_CERTIFICATE: ${{ secrets.MAC_CERTIFICATE }}
+      MAC_CERTIFICATE_PASSWORD: ${{ secrets.MAC_CERTIFICATE_PASSWORD }}
+      SIGNING_APPLE_ID: ${{ secrets.SIGNING_APPLE_ID }}
+      SIGNING_APP_PASSWORD: ${{ secrets.SIGNING_APP_PASSWORD }}
+      SIGNING_TEAM_ID: ${{ secrets.SIGNING_TEAM_ID }}
+    steps:
+      - run: git config --global core.autocrlf false
+
+      - name: Checkout git repo
+        uses: actions/checkout@v4
+
+      - name: Setup & Build
+        uses: ./actions/setup_and_build
         with:
-          MAC_CERTIFICATE: ${{ secrets.MAC_CERTIFICATE }}
-          MAC_CERTIFICATE_PASSWORD: ${{ secrets.MAC_CERTIFICATE_PASSWORD }}
-          SIGNING_APPLE_ID: ${{ secrets.SIGNING_APPLE_ID }}
-          SIGNING_APP_PASSWORD: ${{ secrets.SIGNING_APP_PASSWORD }}
-          SIGNING_TEAM_ID: ${{ secrets.SIGNING_TEAM_ID }}
+          cache_suffix: 'macos_x64'
+
+        # we want to test on all platforms since some are testing the rendered menus (and are dependent on the platform)
+      - name: Unit Test
+        run: yarn test
+
+      - name: Build but do not publish
+        # we want this to run always, except on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'false' }}
+        run: |
+          source ./build/setup-mac-certificate.sh
+          yarn build-release --config.mac.bundleVersion=${{ github.ref }}
+
+      - name: Upload artefacts
+        # we want this to run always, except on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'false' }}
+        uses: ./actions/upload_prod_artefacts
+        with:
+          upload_prefix: ${{ runner.os }}-${{ runner.arch }}
+
+      - name: Build & publish
+        # we want this to run only when on "push" to "master"
+        if: ${{ env.SHOULD_PUBLISH == 'true' }}
+        run: |
+          source ./build/setup-mac-certificate.sh
+          yarn build-release-publish --config.mac.bundleVersion=${{ github.ref }}

CONTRIBUTING.md

@@ -274,3 +274,5 @@ You can build a production binary by running the following:
 yarn build-everything
 yarn build-release
 ```
+
+On linux, you can change in package.json `"target": ["deb"],` to any of the [electron-builder targets](https://www.electron.build/linux#target) to build for another target.

INTERNALBUILDS.md

@@ -146,3 +146,5 @@ yarn build-release
 ```
 
 The binaries will be placed inside the `release/` folder.
+
+On linux, you can change in package.json `"target": ["deb"],` to any of the [electron-builder targets](https://www.electron.build/linux#target) to build for another target.

actions/setup_and_build/action.yml

@@ -1,29 +1,25 @@
-
 name: 'Setup and build'
 description: 'Setup and build Session Desktop'
+inputs:
+  cache_suffix:
+    description: 'the package we are currently building (used as key for the cached node_modules)'
+    required: true
+
 runs:
   using: 'composite'
   steps:
     - name: Install node
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version-file: '.nvmrc'
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: '3.11'
 
-    - name: Cache Desktop node_modules
-      id: cache-desktop-modules
-      uses: actions/cache@v3
-      if: runner.os != 'Windows'
-      with:
-        path: node_modules
-        key: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package.json', 'yarn.lock', 'patches/**') }}
-
     # Not having this will break the windows build because the PATH won't be set by msbuild.
     - name: Add msbuild to PATH
-      uses: microsoft/setup-msbuild@v1.3.1
+      uses: microsoft/setup-msbuild@v2
       if: runner.os == 'Windows'
 
     - name: Setup node for windows
@@ -32,11 +28,24 @@ runs:
       run: |
         yarn global add node-gyp@latest
 
+    - uses: actions/cache/restore@v4
+      id: cache-desktop-modules
+      with:
+        path: node_modules
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache_suffix }}-${{ hashFiles('package.json', 'yarn.lock', 'patches/**') }}
+
     - name: Install dependencies
       shell: bash
       if: steps.cache-desktop-modules.outputs.cache-hit != 'true'
       run: yarn install --frozen-lockfile --network-timeout 600000
 
+    - uses: actions/cache/save@v4
+      id: cache-desktop-modules-save
+      if: runner.os != 'Windows'
+      with:
+        path: node_modules
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache_suffix }}-${{ hashFiles('package.json', 'yarn.lock', 'patches/**') }}
+
     - name: Generate and concat files
       shell: bash
       run: yarn build-everything

actions/upload_prod_artefacts/action.yml

@@ -0,0 +1,27 @@
+name: 'Upload production artefact (not publish)'
+description: 'Upload production artefact (not publish)'
+inputs:
+  upload_prefix:
+    description: 'upload name prefix'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Remove unpacked files
+      run: |
+        ls -d -- */ | xargs -I{} echo "Removing {}"
+        ls -d -- */ | xargs -I{} rm -rf {}
+      shell: bash
+      working-directory: ./release/
+
+    - name: Remaining files
+      run: ls .
+      shell: bash
+      working-directory: ./release/
+
+    - name: Upload Production Artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ inputs.upload_prefix }}-production
+        path: release

build/updateLocalConfig.js

@@ -3,7 +3,7 @@ var _ = require('lodash');
 var execSync = require('child_process').execSync;
 
 const updateLocalConfig = () => {
-  var environment = process.env.SIGNAL_ENV || 'production';
+  var environment = 'production';
   var configPath = `config/local-${environment}.json`;
   var localConfig;