finish testing scubaresults for errors

.github/actions/setup-dependencies-windows/action.yml

@@ -27,12 +27,14 @@ runs:
         pip install pytest
         pip install selenium
 
+        pip uninstall -y numpy
+        pip install numpy==1.26.4
         # Below python v3.9, a lower numpy v1.24.4 is used
-        $pythonVersion = [version]${{ inputs.python-version }}
-        if ($pythonVersion -ge [version]"3.8.18") {
-          pip uninstall -y numpy
-          pip install numpy==1.26.4
-        } 
+        #$pythonVersion = [version]${{ inputs.python-version }}
+        #if ($pythonVersion -ge [version]"3.8.18") {
+        #  pip uninstall -y numpy
+        #  pip install numpy==1.26.4
+        #} 
     
     - name: Download OPA executable
       shell: powershell

Testing/Functional/SmokeTests/smoke_test.py

@@ -4,6 +4,8 @@
 It checks for the following cases:
 - Generate the correct output files (BaselineReports.html, ScubaResults.json, etc)
 - Check the content of html files, verify href attributes are correct, etc
+- Check if ScubaResults.json contains errors in the summary. If errors exist, then 
+  either API calls or functions produced exceptions which need to be handled
 """
 
 import pytest
@@ -16,6 +18,7 @@
     verify_all_outputs_exist, 
     verify_output_type,
     run_selenium,
+    verify_scubaresults,
 )
 class SmokeTest:
     def test_scubagoggles_output(self, subjectemail):
@@ -28,6 +31,15 @@ def test_scubagoggles_output(self, subjectemail):
             verify_all_outputs_exist(output, required_entries)
         except (OSError, ValueError, Exception) as e:
             pytest.fail(f"An error occurred, {e}")
+
+    def test_scubaresults(self):
+        try:
+            output_path: str = get_output_path()
+            scubaresults_path: str = os.path.join(output_path, "ScubaResults.json")
+            with open(scubaresults_path) as jsonfile:
+                verify_scubaresults(jsonfile)
+        except ValueError as e:
+            raise ValueError(f"{scubaresults_path} contains invalid json, {e}")
 
     def test_scubagoggles_report(self, browser, domain):
         try:

Testing/Functional/SmokeTests/smoke_test_utils.py

@@ -59,6 +59,12 @@ def verify_all_outputs_exist(output: list, required_entries: list):
         else:
             raise ValueError(f"{required_entry} was not found in the generated report")
 
+def verify_scubaresults(jsonfile):
+    scubaresults = json.load(jsonfile)
+    summary = scubaresults["Summary"]
+    for v in summary.values():
+        assert v["Errors"] == 0
+
 def run_selenium(browser, domain):
     verify_navigation_links(browser)
     h1 = browser.find_element(By.TAG_NAME, "h1").text