added test_scubagoggles_report pytest with selenium; testing Baseline…

…Reports.html successfully so far

.github/actions/setup-dependencies-macos/action.yml

@@ -23,6 +23,7 @@ runs:
         python -m pip install .
         pip install -r requirements.txt
         pip install pytest
+        pip install selenium
         pip uninstall -y numpy
         pip install numpy==1.26.4
     

.github/actions/setup-dependencies-windows/action.yml

@@ -23,6 +23,7 @@ runs:
         python -m pip install .
         pip install -r requirements.txt
         pip install pytest
+        pip install selenium
         pip uninstall -y numpy
         pip install numpy==1.26.4
     

.github/workflows/run_smoke_test.yml

@@ -58,4 +58,4 @@ jobs:
           json: ${{ secrets.GWS_GITHUB_AUTOMATION_CREDS }}
 
       - name: Execute ScubaGoggles and check for correct output
-        run: pytest -s ./Testing/Functional/SmokeTests/ --subjectemail="${{ secrets.GWS_SUBJECT_EMAIL }}"
+        run: pytest -s ./Testing/Functional/SmokeTests/ --subjectemail="${{ secrets.GWS_SUBJECT_EMAIL }}" --domain="${{ secrets.GWS_DOMAIN }}"

Testing/Functional/SmokeTests/smoke_test.py

@@ -6,23 +6,77 @@
 import pytest
 import subprocess
 import os
-
-from smoke_test_utils import verify_all_outputs_exist, verify_output_type
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions
+from smoke_test_utils import get_output_path, verify_all_outputs_exist, verify_output_type
 
 class SmokeTest:
     def test_scubagoggles_output(self, subjectemail):
         try:
             command: str = f"scubagoggles gws --subjectemail {subjectemail} --quiet"
             subprocess.run(command, shell=True)
 
-            prefix: str = "GWSBaselineConformance"
-            directories: list[str] = [d for d in os.listdir() if os.path.isdir(d) and d.startswith(prefix)]
-            directories.sort(key=lambda d: os.path.getctime(d), reverse=True)
-
-            # Access the latest output directory at the 0th index after sorting
-            cwd: str = os.getcwd()
-            output_path: str = os.path.join(cwd, directories[0])
+            output_path: str = get_output_path()
             contents: list[str] = verify_output_type(output_path, [])
             verify_all_outputs_exist(contents)
         except (OSError, ValueError, Exception) as e:
             pytest.fail(f"An error occurred, {e}")
+
+    def test_scubagoggles_report(self, browser, domain):
+        try:
+            output_path: str = get_output_path()
+            browser.get(os.path.join(output_path, "BaselineReports.html"))
+
+            h1 = browser.find_element(By.TAG_NAME, "h1").text
+            assert h1 == "SCuBA GWS Security Baseline Conformance Reports"
+
+            # verify domain is present in agency table
+            agency_table = browser.find_element(By.TAG_NAME, "table")
+            tbody = agency_table.find_element(By.TAG_NAME, "tbody")
+            rows = tbody.find_elements(By.TAG_NAME, "tr")
+            assert len(rows) == 2
+            assert rows[1].find_elements(By.TAG_NAME, "td")[0].text == domain
+
+            baseline_names = [
+                "Google Calendar", 
+                "Google Chat", 
+                "Google Classroom", 
+                "Common Controls", 
+                "Google Drive and Docs",
+                "Gmail",
+                "Groups for Business",
+                "Google Meet",
+                "Rules",
+                "Google Sites"
+            ]
+
+            for i in range(10): 
+                reports_table = browser.find_elements(By.TAG_NAME, "table")[1]
+                tbody = reports_table.find_element(By.TAG_NAME, "tbody")
+                rows = tbody.find_elements(By.TAG_NAME, "tr")
+                td = rows[i].find_elements(By.TAG_NAME, "td")[0]
+                assert td.text in baseline_names
+
+                individual_report_anchor = td.find_element(By.TAG_NAME, "a")
+                href = individual_report_anchor.get_attribute("href")
+                individual_report_anchor.get_attribute("href").click()
+                current_url = browser.current_url
+                assert href == current_url
+
+                browser.back()
+                WebDriverWait(browser, 10).until(
+                    expected_conditions.presence_of_element_located(
+                        (By.TAG_NAME, "body")
+                    )
+                )
+
+            # Navigation to detailed reports 
+
+            # Check links work
+
+            # Verify tables are populated
+
+
+        except Exception as e:
+            pytest.fail(f"An error occurred, {e}")

Testing/Functional/SmokeTests/smoke_test_utils.py

@@ -2,25 +2,10 @@
 import os
 import json
 
-required_contents = [
-    "BaselineReports.html", 
-    "IndividualReports", 
-    "ScubaResults.json",
-    "TestResults.json",
-    "images",
-    "CalendarReport.html",
-    "ChatReport.html",
-    "ClassroomReport.html",
-    "CommoncontrolsReport.html",
-    "DriveReport.html",
-    "GmailReport.html",
-    "GroupsReport.html",
-    "MeetReport.html",
-    "RulesReport.html",
-    "SitesReport.html",
-    "cisa_logo.png",
-    "triangle-exclamation-solid.svg"
-]
+def get_output_path() -> str:
+    directories: list[str] = [d for d in os.listdir() if os.path.isdir(d) and d.startswith("GWSBaselineConformance")]
+    directories.sort(key=lambda d: os.path.getctime(d), reverse=True)
+    return os.path.join(os.getcwd(), directories[0])
 
 def verify_output_type(output_path: str, contents: list[str]) -> list[str]:
     entries: list[str] = os.listdir(output_path)
@@ -44,6 +29,26 @@ def verify_output_type(output_path: str, contents: list[str]) -> list[str]:
             raise OSError(f"Entry is not a directory or file (symlink, etc.)")
     return contents
 
+required_contents = [
+    "BaselineReports.html", 
+    "IndividualReports", 
+    "ScubaResults.json",
+    "TestResults.json",
+    "images",
+    "CalendarReport.html",
+    "ChatReport.html",
+    "ClassroomReport.html",
+    "CommoncontrolsReport.html",
+    "DriveReport.html",
+    "GmailReport.html",
+    "GroupsReport.html",
+    "MeetReport.html",
+    "RulesReport.html",
+    "SitesReport.html",
+    "cisa_logo.png",
+    "triangle-exclamation-solid.svg"
+]
+
 def verify_all_outputs_exist(contents: list[str]):
     for required_content in required_contents:
         if required_content in contents:

Testing/Functional/conftest.py

@@ -1,8 +1,20 @@
 import pytest
+from selenium import webdriver
 
 def pytest_addoption(parser):
     parser.addoption("--subjectemail", action="store")
+    parser.addoption("--domain", action="store")
 
 @pytest.fixture 
 def subjectemail(pytestconfig):
-    return pytestconfig.getoption("subjectemail")
+    return pytestconfig.getoption("subjectemail")
+
+@pytest.fixture
+def domain(pytestconfig):
+    return pytestconfig.getoption("domain")
+
+@pytest.fixture
+def browser():
+    driver = webdriver.Chrome()
+    yield driver
+    driver.quit()