conditionally uninstall numpy to see if 3.8.10 passes smoke test; som…

…e additional refactoring/code improvements

.github/actions/setup-dependencies-macos/action.yml

@@ -31,8 +31,4 @@ runs:
       shell: bash
       run: |
         python download_opa.py -v ${{ inputs.opa-version }} -os ${{ inputs.operating-system }}
-
-        # Give OPA executable execute permissions
-        chmod +x opa_darwin_amd64
-    
-
+        chmod +x opa_darwin_amd64

.github/actions/setup-dependencies-windows/action.yml

@@ -6,6 +6,8 @@ inputs:
   opa-version:
     required: true
     default: "0.60.0"
+  python-version:
+    required: true
 
 runs:
   using: "composite"
@@ -24,8 +26,13 @@ runs:
         pip install -r requirements.txt
         pip install pytest
         pip install selenium
-        pip uninstall -y numpy
-        pip install numpy==1.26.4
+
+        # Below python v3.9, a lower numpy v1.24.4 is used
+        $pythonVersion = [version]${{ inputs.python-version }}
+        if ($pythonVersion -ge [version]"3.9") {
+          pip uninstall -y numpy
+          pip install numpy==1.26.4
+        } 
     
     - name: Download OPA executable
       shell: powershell

.github/workflows/run_smoke_test.yml

@@ -45,13 +45,15 @@ jobs:
         with:
           operating-system: "windows"
           opa-version: "0.60.0"
+          python-version: ${{ matrix.python-version }}
 
       - name: Setup Dependencies (macOS)
         if: ${{ matrix.os == 'macos-latest' }}
         uses: ./.github/actions/setup-dependencies-macos
         with:
           operating-system: "macos"
           opa-version: "0.60.0"
+          python-version: ${{ matrix.python-version }}
 
       - name: Setup credentials for service account
         id: create-json

Testing/Functional/SmokeTests/smoke_test.py

@@ -12,6 +12,7 @@
 from smoke_test_utils import (
     get_output_path, 
     prepend_file_protocol, 
+    get_required_entries,
     verify_all_outputs_exist, 
     verify_output_type,
     run_selenium,
@@ -21,10 +22,10 @@ def test_scubagoggles_output(self, subjectemail):
         try:
             command: str = f"scubagoggles gws --subjectemail {subjectemail} --quiet"
             subprocess.run(command, shell=True)
-
             output_path: str = get_output_path()
             output: list[str] = verify_output_type(output_path, [])
-            verify_all_outputs_exist(output)
+            required_entries = get_required_entries(os.path.join(os.getcwd(), "sample-report"), [])
+            verify_all_outputs_exist(output, required_entries)
         except (OSError, ValueError, Exception) as e:
             pytest.fail(f"An error occurred, {e}")
 
@@ -34,6 +35,6 @@ def test_scubagoggles_report(self, browser, domain):
             report_path: str = prepend_file_protocol(os.path.join(output_path, "BaselineReports.html"))
             browser.get(report_path)
             run_selenium(browser, domain)
-        except Exception as e:
+        except (ValueError, Exception) as e:
             browser.quit()
             pytest.fail(f"An error occurred, {e}")

Testing/Functional/SmokeTests/smoke_test_utils.py

@@ -6,6 +6,9 @@
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions
 
+CISA_GOV_URL = "https://www.cisa.gov/scuba"
+SCUBAGOGGLES_BASELINES_URL = "https://github.com/cisagov/ScubaGoggles/tree/main/baselines"
+
 def get_output_path() -> str:
     directories: list[str] = [d for d in os.listdir() if os.path.isdir(d) and d.startswith("GWSBaselineConformance")]
     directories.sort(key=lambda d: os.path.getctime(d), reverse=True)
@@ -20,13 +23,15 @@ def verify_output_type(output_path: str, output: list[str]) -> list[str]:
     entries: list[str] = os.listdir(output_path)
     for entry in entries:
         output.append(entry)
+
         # Check if entry is a valid directory or file
         # If a valid directory, then recurse
         child_path: str = os.path.join(output_path, entry)
         if os.path.isdir(child_path):
             assert True
             verify_output_type(child_path, output)
         elif os.path.isfile(child_path):
+
             # Check for valid json
             if child_path.endswith(".json"):
                 try:
@@ -39,53 +44,42 @@ def verify_output_type(output_path: str, output: list[str]) -> list[str]:
             raise OSError(f"Entry is not a directory or file (symlink, etc.)")
     return output
 
-required_entries = [
-    "BaselineReports.html", 
-    "IndividualReports", 
-    "ScubaResults.json",
-    "TestResults.json",
-    "images",
-    "CalendarReport.html",
-    "ChatReport.html",
-    "ClassroomReport.html",
-    "CommoncontrolsReport.html",
-    "DriveReport.html",
-    "GmailReport.html",
-    "GroupsReport.html",
-    "MeetReport.html",
-    "RulesReport.html",
-    "SitesReport.html",
-    "cisa_logo.png",
-    "triangle-exclamation-solid.svg"
-]
-
-def verify_all_outputs_exist(output: list[str]):
+def get_required_entries(directory, required_entries) -> list[str]:
+    with os.scandir(directory) as entries:
+        for entry in entries:
+            required_entries.append(entry.name)
+            if entry.is_dir():
+                get_required_entries(entry.path, required_entries)
+    return required_entries
+
+def verify_all_outputs_exist(output: list[str], required_entries: list[str]):
     for required_entry in required_entries:
         if required_entry in output:
             assert True
         else:
             raise ValueError(f"{required_entry} was not found in the generated report")
 
 def run_selenium(browser, domain):
+    verify_navigation_links(browser)
     h1 = browser.find_element(By.TAG_NAME, "h1").text
     assert h1 == "SCuBA GWS Security Baseline Conformance Reports"
 
     products = {
         product: { "title": f"{product} Baseline Report" }
         for product in gws_products()["prod_to_fullname"].values()
     }
-    print(products)
 
     # Before entering loop check that we actually display 10 rows in table
     reports_table = get_reports_table(browser)
 
     if len(reports_table) == 10:
         for i in range(len(reports_table)): 
-            # Verify domain is present in agency table
-            tenant_table = get_tenant_table(browser)
-            assert len(tenant_table) == 2
-            customer_domain = tenant_table[1].find_elements(By.TAG_NAME, "td")[0].text
-            assert customer_domain == domain
+
+            # Check if domain is present in agency table
+            verify_tenant_table(browser, domain)
+
+            # TODO:
+            # Check for correct baseline and tool version, e.g. 0.2, 0.2.0
 
             reports_table = get_reports_table(browser)[i]
             baseline_report = reports_table.find_elements(By.TAG_NAME, "td")[0]
@@ -99,17 +93,15 @@ def run_selenium(browser, domain):
             assert href == current_url
 
             # Check at the individual report level
-            tenant_table = get_tenant_table(browser)
-            assert len(tenant_table) == 2
-            assert tenant_table[1].find_elements(By.TAG_NAME, "td")[0].text == domain
-
+            verify_navigation_links(browser)
             h1 = browser.find_element(By.TAG_NAME, "h1").text
-            print(products[product])
-            print(products[product]["title"])
             assert h1 == products[product]["title"]
 
+            verify_tenant_table(browser, domain)
+
             policy_tables = browser.find_elements(By.TAG_NAME, "table")
             for table in policy_tables[1:]:
+
                 # Verify policy table headers are correct 
                 headers = (
                     table.find_element(By.TAG_NAME, "thead")
@@ -137,16 +129,28 @@ def run_selenium(browser, domain):
     else:
         raise ValueError(f"Expected the reports table to have a length of 10")
 
-def get_tenant_table(browser):
-    return (
-        browser.find_element(By.TAG_NAME, "table")
-        .find_element(By.TAG_NAME, "tbody")
-        .find_elements(By.TAG_NAME, "tr")
+def verify_navigation_links(browser):
+    links = (
+        browser.find_element(By.CLASS_NAME, "links")
+        .find_elements(By.TAG_NAME, "a")
     )
+    if len(links) == 2:
+        assert links[0].get_attribute("href") == CISA_GOV_URL
+        assert links[1].get_attribute("href") == SCUBAGOGGLES_BASELINES_URL
 
 def get_reports_table(browser):
     return (
         browser.find_elements(By.TAG_NAME, "table")[1]
         .find_element(By.TAG_NAME, "tbody")
         .find_elements(By.TAG_NAME, "tr")
-    )
+    )
+
+def verify_tenant_table(browser, domain):
+    tenant_table = (
+        browser.find_element(By.TAG_NAME, "table")
+        .find_element(By.TAG_NAME, "tbody")
+        .find_elements(By.TAG_NAME, "tr")
+    )
+    assert len(tenant_table) == 2
+    customer_domain = tenant_table[1].find_elements(By.TAG_NAME, "td")[0].text
+    assert customer_domain == domain