From 2e608aef1a227b3624ba13d22cac6c7090f73ea2 Mon Sep 17 00:00:00 2001
From: Kyle Harding <kyle@balena.io>
Date: Wed, 19 Jun 2024 15:49:25 -0400
Subject: [PATCH] Allow both combinatorial and include syntax for test_matrix

The include syntax allows full control over settings used
in each test job, but the combinatorial is simpler to write
for basic use cases.

Rather than force one, we can allow both and default to the simple
syntax.

Change-type: patch
Signed-off-by: Kyle Harding <kyle@balena.io>
---
 .github/workflows/generic-amd64.yml      | 23 +++---------
 .github/workflows/yocto-build-deploy.yml | 48 +++++++++++++++---------
 2 files changed, 36 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/generic-amd64.yml b/.github/workflows/generic-amd64.yml
index cc75476ce..b64fdf314 100644
--- a/.github/workflows/generic-amd64.yml
+++ b/.github/workflows/generic-amd64.yml
@@ -53,21 +53,8 @@ jobs:
       device-repo-ref: master
       # Use qemu workers for testing
       test_matrix: >
-        [
-          {
-            "test_suite": "os",
-            "environment": "bm.balena-dev.com",
-            "worker_type": "qemu"
-          },
-          {
-            "test_suite": "cloud",
-            "environment": "bm.balena-dev.com",
-            "worker_type": "qemu",
-            "test_org": "testbot"
-          },
-          {
-            "test_suite": "hup",
-            "environment": "bm.balena-dev.com",
-            "worker_type": "qemu"
-          },
-        ]
+        {
+          "test_suite": ["os","cloud","hup"],
+          "environment": ["bm.balena-dev.com"],
+          "worker_type": ["qemu"]
+        }
diff --git a/.github/workflows/yocto-build-deploy.yml b/.github/workflows/yocto-build-deploy.yml
index 49b6e1529..f55d4dee6 100644
--- a/.github/workflows/yocto-build-deploy.yml
+++ b/.github/workflows/yocto-build-deploy.yml
@@ -92,6 +92,8 @@ on:
         required: false
         type: boolean
         default: ${{ github.event_name == 'push' }} # This determines if we want to check the results of the merge PR - we only want to do it when a new tag is made
+      # FIXME: remove this input and set "test_matrix: {}" to skip tests. The open PR event type check can be moved to the job conditions.
+      # Additionally, we may want to change the default test_matrix to {} and make tests opt-in.
       run-tests:
         required: false
         type: boolean
@@ -134,25 +136,38 @@ on:
       # - worker_type: The worker type to use for testing. The valid worker types are `qemu` and `testbot`. The default worker type is `testbot`
       # - worker_fleets: The testbot fleets for finding available Leviathan workers. Not used for QEMU workers. Can accept a list of apps separated by commas, no spaces in between
       # - test_org: The organization to use for testing cloud functionality. This default org is `testbot`
+      # To use specific settings for each test job, create an include array like this...
+      # {"include": [
+      #   {
+      #     "test_suite": "os",
+      #     "environment": "bm.balena-dev.com"
+      #   },
+      #   {
+      #     "test_suite": "cloud",
+      #     "environment": "balena-cloud.com",
+      #     "worker_type": "qemu"
+      #   },
+      #   {
+      #     "test_suite": "hup",
+      #     "environment": "balena-cloud.com",
+      #     "worker_type": "qemu"
+      #   }
+      # ]}
+      # Otherwise for a combinatorial matrix (the default) just provide values like this to have the matrix expanded...
+      # {
+      #   "test_suite": ["os","cloud","hup"],
+      #   "environment": ["bm.balena-dev.com"],
+      #   "worker_type": ["qemu","testbot"]
+      # }
       test_matrix:
-        description: "JSON Leviathan test matrix to use for testing. If not provided, the default test matrix will be used."
+        description: "JSON Leviathan test matrix to use for testing. If not provided, the default testbot matrix will be used."
         required: false
         type: string
         default: >
-          [
-            {
-              "test_suite": "os",
-              "environment": "bm.balena-dev.com"
-            },
-            {
-              "test_suite": "cloud",
-              "environment": "bm.balena-dev.com"
-            },
-            {
-              "test_suite": "hup",
-              "environment": "bm.balena-dev.com"
-            },
-          ]
+          {
+            "test_suite": ["os","cloud","hup"],
+            "environment": ["bm.balena-dev.com"]
+          }
 
 # https://docs.github.com/en/actions/using-jobs/using-concurrency
 concurrency:
@@ -978,8 +993,7 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix:
-        include: ${{ fromJSON(inputs.test_matrix) }}
+      matrix: ${{ fromJSON(inputs.test_matrix) }}
 
     env:
       # Variables provided via the selected GitHub environment