Add functional smoke tests and automated testing workflow

.github/actions/setup-dependencies-macos/action.yml

@@ -0,0 +1,36 @@
+name: Setup Dependencies (macOS)
+inputs:
+  operating-system:
+    required: true
+    default: "macos"
+  opa-version:
+    required: true
+    default: "0.60.0"
+  python-version:
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup virtualenv
+      shell: bash
+      run: |
+        pip install virtualenv
+        virtualenv -p python .venv
+        source .venv/bin/activate
+
+    - name: Install dependencies
+      shell: bash
+      run: |
+        python -m pip install .
+        pip install -r requirements.txt
+        pip install pytest
+        pip install selenium
+        pip uninstall -y numpy
+        pip install numpy==1.26.4
+
+    - name: Download OPA executable
+      shell: bash
+      run: |
+        python download_opa.py -v ${{ inputs.opa-version }} -os ${{ inputs.operating-system }}
+        chmod +x opa_darwin_amd64

.github/actions/setup-dependencies-windows/action.yml

@@ -0,0 +1,41 @@
+name: Setup Dependencies (Windows)
+inputs:
+  operating-system:
+    required: true
+    default: "windows"
+  opa-version:
+    required: true
+    default: "0.60.0"
+  python-version:
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup virtualenv
+      shell: powershell
+      run: |
+        pip install virtualenv
+        python -m venv .venv
+        .venv\Scripts\activate
+
+    - name: Install dependencies
+      shell: powershell
+      run: |
+        python -m pip install .
+        pip install -r requirements.txt
+        pip install pytest
+        pip install selenium
+        pip uninstall -y numpy
+        pip install numpy==1.26.4
+
+        # Below python v3.9, a lower numpy v1.24.4 is used
+        #$pythonVersion = [version]${{ inputs.python-version }}
+        #if ($pythonVersion -ge [version]"3.8.18") {
+        #  pip uninstall -y numpy
+        #  pip install numpy==1.26.4
+        #} 
+
+    - name: Download OPA executable
+      shell: powershell
+      run: python download_opa.py -v ${{ inputs.opa-version }} -os ${{ inputs.operating-system }}

.github/workflows/pylint.yml

@@ -31,7 +31,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install pylint
+        pip install pylint pytest selenium
     - name: Analysing the code with pylint
       run: |
         pylint -d R0913,R0914,R0915,R1702,W0718,W0719,R0801 $(git ls-files '*.py')

.github/workflows/run_smoke_test.yml

@@ -0,0 +1,67 @@
+name: Run Smoke Test 
+on: 
+  workflow_call:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, reopened, synchronize]
+    branches:
+      - "main"
+  pull_request_review:
+    types: [submitted]
+  push:
+    # Uncomment when testing locally
+    #paths:
+    #  - ".github/workflows/run_smoke_test.yml"
+    #  - ".github/actions/setup-dependencies-windows/action.yml"
+    #  - ".github/actions/setup-dependencies-macos/action.yml"
+    branches:
+      - "main"
+      #- "*smoke*"
+
+jobs:
+  smoke-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, macos-latest]
+        # See https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json,
+        # ctrl + f and search "python-3.<minor>.<patch>-<darwin-arm64/win32/linux>" for supported versions
+        python-version: ["3.9", "3.12"] # "3.8 fails with numpy uninstall"
+    runs-on: ${{ matrix.os }}
+    environment: Development
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python v${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with: 
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: "requirements.txt"
+
+      - name: Setup Dependencies (Windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        uses: ./.github/actions/setup-dependencies-windows
+        with:
+          operating-system: "windows"
+          opa-version: "0.60.0"
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup Dependencies (macOS)
+        if: ${{ matrix.os == 'macos-latest' }}
+        uses: ./.github/actions/setup-dependencies-macos
+        with:
+          operating-system: "macos"
+          opa-version: "0.60.0"
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup credentials for service account
+        id: create-json
+        uses: jsdaniell/[email protected]
+        with: 
+          name: "credentials.json"
+          json: ${{ secrets.GWS_GITHUB_AUTOMATION_CREDS }}
+
+      - name: Run ScubaGoggles and check for correct output
+        run: pytest -s -vvv ./Testing/Functional/SmokeTests/ --subjectemail="${{ secrets.GWS_SUBJECT_EMAIL }}" --domain="${{ secrets.GWS_DOMAIN }}"

Testing/Functional/SmokeTests/selenium_browser.py

@@ -0,0 +1,75 @@
+"""
+selenium_browser.py declares a Browser class for use in ScubaGoggles testing.    
+"""
+
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+
+class Browser:
+    """
+    The Browser class encapsulates the setup, usage, and teardown of a
+    Selenium WebDriver instance for automated browser interactions.
+    """
+    def __init__(self):
+        chrome_options = Options()
+        chrome_options.add_argument("--headless")
+        chrome_options.add_argument("--no-sandbox")
+        chrome_options.add_argument("--disable-dev-shm-usage")
+
+        self.driver = webdriver.Chrome(options=chrome_options)
+
+    def get(self, url):
+        """
+        Load a new web page in the current browser window.
+
+        Args:
+            url: The URL to load. Must be a fully qualified URL
+        """
+        self.driver.get(url)
+
+    def quit(self):
+        """
+        Quits the driver, closing every associated window. 
+        """
+        self.driver.quit()
+
+    def find_element(self, by, value):
+        """
+        Find the first WebElement using the given method.
+        This method is affected by the 'implicit wait' times in force at the time and execution.
+        The find_element(...) invocation will return a matching row, or try again repeatedly
+        until the configured timeout is reached.
+
+        Args:
+            by: The locating mechanism to use, i.e. By.CLASS_NAME, By.TAG_NAME
+            value: The locator, i.e. "h1", "header"
+
+        Returns:
+            WebElement: The first matching element on the current page
+        """
+        return self.driver.find_element(by, value)
+
+    def find_elements(self, by, value):
+        """
+        Find all WebElements within the current page.
+        This method is affected by the 'implicit wait' times in force at the time and execution.
+        The find_elements(...) invocation will return as soon as there are more than 0 items in 
+        the found collection, or will return an empty list of the timeout is reached.
+
+        Args:
+            by: The locating mechanism to use, i.e. By.CLASS_NAME, By.TAG_NAME
+            value: The locator, i.e. "h1", "header"
+
+        Returns:
+            WebElement: A list of all matching WebElements, or an empty list if nothing matches
+        """
+        return self.driver.find_elements(by, value)
+
+    def current_url(self):
+        """
+        Get a string representing the current URL that the browser is looking at.
+
+        Returns:
+            The URL of the page currently loaded in the browser
+        """
+        return self.driver.current_url

Testing/Functional/SmokeTests/smoke_test.py

@@ -0,0 +1,72 @@
+"""
+smoke_test.py declares a SmokeTest class for ScubaGoggles automation testing.
+"""
+
+import subprocess
+import os
+import pytest
+from smoke_test_utils import (
+    get_output_path,
+    prepend_file_protocol,
+    get_required_entries,
+    verify_all_outputs_exist,
+    verify_output_type,
+    run_selenium,
+    verify_scubaresults,
+)
+
+SAMPLE_REPORT = "sample-report"
+SCUBA_RESULTS = "ScubaResults.json"
+BASELINE_REPORTS = "BaselineReports.html"
+
+class SmokeTest:
+    """
+    Pytest class to encapsulate the following test cases:
+
+    - Generate the correct output files (BaselineReports.html, ScubaResults.json, etc)
+    - Check the content of html files, verify href attributes are correct, etc
+    - Check if ScubaResults.json contains errors in the summary. If errors exist, then
+      either API calls or functions produced exceptions which need to be handled
+    """
+    def test_scubagoggles_output(self, subjectemail):
+        """
+        Test if the `scubagoggles gws` command generates correct output for all baselines.
+
+        Args:
+            subjectemail: The email address of a user for the service account
+        """
+        try:
+            command: str = f"scubagoggles gws --subjectemail {subjectemail} --quiet"
+            subprocess.run(command, shell=True, check=True)
+            output_path: str = get_output_path()
+            output: list = verify_output_type(output_path, [])
+            required_entries = get_required_entries(os.path.join(os.getcwd(), SAMPLE_REPORT), [])
+            verify_all_outputs_exist(output, required_entries)
+        except (OSError, ValueError, Exception) as e:
+            pytest.fail(f"An error occurred, {e}")
+
+    def test_scubaresults(self):
+        """
+        Determine if ScubaResults.json contains API errors or exceptions.
+        """
+        try:
+            output_path: str = get_output_path()
+            scubaresults_path: str = os.path.join(output_path, SCUBA_RESULTS)
+            with open(scubaresults_path, encoding="utf-8") as jsonfile:
+                verify_scubaresults(jsonfile)
+        except (ValueError, Exception) as e:
+            pytest.fail(f"An error occurred, {e}")
+
+    def test_scubagoggles_report(self, browser, domain):
+        """
+        Test if the generated baseline reports are correct,
+        i.e. BaselineReports.html, CalendarReport.html, ChatReport.html
+        """
+        try:
+            output_path: str = get_output_path()
+            report_path: str = prepend_file_protocol(os.path.join(output_path, BASELINE_REPORTS))
+            browser.get(report_path)
+            run_selenium(browser, domain)
+        except (ValueError, AssertionError, Exception) as e:
+            browser.quit()
+            pytest.fail(f"An error occurred, {e}")