diff --git a/.github/actions/setup-dependencies-macos/action.yml b/.github/actions/setup-dependencies-macos/action.yml new file mode 100644 index 00000000..06eedebe --- /dev/null +++ b/.github/actions/setup-dependencies-macos/action.yml @@ -0,0 +1,36 @@ +name: Setup Dependencies (macOS) +inputs: + operating-system: + required: true + default: "macos" + opa-version: + required: true + default: "0.60.0" + python-version: + required: true + +runs: + using: "composite" + steps: + - name: Setup virtualenv + shell: bash + run: | + pip install virtualenv + virtualenv -p python .venv + source .venv/bin/activate + + - name: Install dependencies + shell: bash + run: | + python -m pip install . + pip install -r requirements.txt + pip install pytest + pip install selenium + pip uninstall -y numpy + pip install numpy==1.26.4 + + - name: Download OPA executable + shell: bash + run: | + python download_opa.py -v ${{ inputs.opa-version }} -os ${{ inputs.operating-system }} + chmod +x opa_darwin_amd64 \ No newline at end of file diff --git a/.github/actions/setup-dependencies-windows/action.yml b/.github/actions/setup-dependencies-windows/action.yml new file mode 100644 index 00000000..771ee2a8 --- /dev/null +++ b/.github/actions/setup-dependencies-windows/action.yml @@ -0,0 +1,41 @@ +name: Setup Dependencies (Windows) +inputs: + operating-system: + required: true + default: "windows" + opa-version: + required: true + default: "0.60.0" + python-version: + required: true + +runs: + using: "composite" + steps: + - name: Setup virtualenv + shell: powershell + run: | + pip install virtualenv + python -m venv .venv + .venv\Scripts\activate + + - name: Install dependencies + shell: powershell + run: | + python -m pip install . + pip install -r requirements.txt + pip install pytest + pip install selenium + pip uninstall -y numpy + pip install numpy==1.26.4 + + # Below python v3.9, a lower numpy v1.24.4 is used + #$pythonVersion = [version]${{ inputs.python-version }} + #if ($pythonVersion -ge [version]"3.8.18") { + # pip uninstall -y numpy + # pip install numpy==1.26.4 + #} + + - name: Download OPA executable + shell: powershell + run: python download_opa.py -v ${{ inputs.opa-version }} -os ${{ inputs.operating-system }} \ No newline at end of file diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index b345351a..14ab44f0 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -31,7 +31,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt - pip install pylint + pip install pylint pytest selenium - name: Analysing the code with pylint run: | pylint -d R0913,R0914,R0915,R1702,W0718,W0719,R0801 $(git ls-files '*.py') diff --git a/.github/workflows/run_smoke_test.yml b/.github/workflows/run_smoke_test.yml new file mode 100644 index 00000000..7ce74d2d --- /dev/null +++ b/.github/workflows/run_smoke_test.yml @@ -0,0 +1,67 @@ +name: Run Smoke Test +on: + workflow_call: + workflow_dispatch: + pull_request: + types: [opened, reopened, synchronize] + branches: + - "main" + pull_request_review: + types: [submitted] + push: + # Uncomment when testing locally + #paths: + # - ".github/workflows/run_smoke_test.yml" + # - ".github/actions/setup-dependencies-windows/action.yml" + # - ".github/actions/setup-dependencies-macos/action.yml" + branches: + - "main" + #- "*smoke*" + +jobs: + smoke-test: + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + # See https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json, + # ctrl + f and search "python-3..-" for supported versions + python-version: ["3.9", "3.12"] # "3.8 fails with numpy uninstall" + runs-on: ${{ matrix.os }} + environment: Development + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup Python v${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + cache-dependency-path: "requirements.txt" + + - name: Setup Dependencies (Windows) + if: ${{ matrix.os == 'windows-latest' }} + uses: ./.github/actions/setup-dependencies-windows + with: + operating-system: "windows" + opa-version: "0.60.0" + python-version: ${{ matrix.python-version }} + + - name: Setup Dependencies (macOS) + if: ${{ matrix.os == 'macos-latest' }} + uses: ./.github/actions/setup-dependencies-macos + with: + operating-system: "macos" + opa-version: "0.60.0" + python-version: ${{ matrix.python-version }} + + - name: Setup credentials for service account + id: create-json + uses: jsdaniell/create-json@v1.2.3 + with: + name: "credentials.json" + json: ${{ secrets.GWS_GITHUB_AUTOMATION_CREDS }} + + - name: Run ScubaGoggles and check for correct output + run: pytest -s -vvv ./Testing/Functional/SmokeTests/ --subjectemail="${{ secrets.GWS_SUBJECT_EMAIL }}" --domain="${{ secrets.GWS_DOMAIN }}" diff --git a/Testing/Functional/SmokeTests/selenium_browser.py b/Testing/Functional/SmokeTests/selenium_browser.py new file mode 100644 index 00000000..69bf9d9d --- /dev/null +++ b/Testing/Functional/SmokeTests/selenium_browser.py @@ -0,0 +1,75 @@ +""" +selenium_browser.py declares a Browser class for use in ScubaGoggles testing. +""" + +from selenium import webdriver +from selenium.webdriver.chrome.options import Options + +class Browser: + """ + The Browser class encapsulates the setup, usage, and teardown of a + Selenium WebDriver instance for automated browser interactions. + """ + def __init__(self): + chrome_options = Options() + chrome_options.add_argument("--headless") + chrome_options.add_argument("--no-sandbox") + chrome_options.add_argument("--disable-dev-shm-usage") + + self.driver = webdriver.Chrome(options=chrome_options) + + def get(self, url): + """ + Load a new web page in the current browser window. + + Args: + url: The URL to load. Must be a fully qualified URL + """ + self.driver.get(url) + + def quit(self): + """ + Quits the driver, closing every associated window. + """ + self.driver.quit() + + def find_element(self, by, value): + """ + Find the first WebElement using the given method. + This method is affected by the 'implicit wait' times in force at the time and execution. + The find_element(...) invocation will return a matching row, or try again repeatedly + until the configured timeout is reached. + + Args: + by: The locating mechanism to use, i.e. By.CLASS_NAME, By.TAG_NAME + value: The locator, i.e. "h1", "header" + + Returns: + WebElement: The first matching element on the current page + """ + return self.driver.find_element(by, value) + + def find_elements(self, by, value): + """ + Find all WebElements within the current page. + This method is affected by the 'implicit wait' times in force at the time and execution. + The find_elements(...) invocation will return as soon as there are more than 0 items in + the found collection, or will return an empty list of the timeout is reached. + + Args: + by: The locating mechanism to use, i.e. By.CLASS_NAME, By.TAG_NAME + value: The locator, i.e. "h1", "header" + + Returns: + WebElement: A list of all matching WebElements, or an empty list if nothing matches + """ + return self.driver.find_elements(by, value) + + def current_url(self): + """ + Get a string representing the current URL that the browser is looking at. + + Returns: + The URL of the page currently loaded in the browser + """ + return self.driver.current_url diff --git a/Testing/Functional/SmokeTests/smoke_test.py b/Testing/Functional/SmokeTests/smoke_test.py new file mode 100644 index 00000000..8c9c6839 --- /dev/null +++ b/Testing/Functional/SmokeTests/smoke_test.py @@ -0,0 +1,72 @@ +""" +smoke_test.py declares a SmokeTest class for ScubaGoggles automation testing. +""" + +import subprocess +import os +import pytest +from smoke_test_utils import ( + get_output_path, + prepend_file_protocol, + get_required_entries, + verify_all_outputs_exist, + verify_output_type, + run_selenium, + verify_scubaresults, +) + +SAMPLE_REPORT = "sample-report" +SCUBA_RESULTS = "ScubaResults.json" +BASELINE_REPORTS = "BaselineReports.html" + +class SmokeTest: + """ + Pytest class to encapsulate the following test cases: + + - Generate the correct output files (BaselineReports.html, ScubaResults.json, etc) + - Check the content of html files, verify href attributes are correct, etc + - Check if ScubaResults.json contains errors in the summary. If errors exist, then + either API calls or functions produced exceptions which need to be handled + """ + def test_scubagoggles_output(self, subjectemail): + """ + Test if the `scubagoggles gws` command generates correct output for all baselines. + + Args: + subjectemail: The email address of a user for the service account + """ + try: + command: str = f"scubagoggles gws --subjectemail {subjectemail} --quiet" + subprocess.run(command, shell=True, check=True) + output_path: str = get_output_path() + output: list = verify_output_type(output_path, []) + required_entries = get_required_entries(os.path.join(os.getcwd(), SAMPLE_REPORT), []) + verify_all_outputs_exist(output, required_entries) + except (OSError, ValueError, Exception) as e: + pytest.fail(f"An error occurred, {e}") + + def test_scubaresults(self): + """ + Determine if ScubaResults.json contains API errors or exceptions. + """ + try: + output_path: str = get_output_path() + scubaresults_path: str = os.path.join(output_path, SCUBA_RESULTS) + with open(scubaresults_path, encoding="utf-8") as jsonfile: + verify_scubaresults(jsonfile) + except (ValueError, Exception) as e: + pytest.fail(f"An error occurred, {e}") + + def test_scubagoggles_report(self, browser, domain): + """ + Test if the generated baseline reports are correct, + i.e. BaselineReports.html, CalendarReport.html, ChatReport.html + """ + try: + output_path: str = get_output_path() + report_path: str = prepend_file_protocol(os.path.join(output_path, BASELINE_REPORTS)) + browser.get(report_path) + run_selenium(browser, domain) + except (ValueError, AssertionError, Exception) as e: + browser.quit() + pytest.fail(f"An error occurred, {e}") diff --git a/Testing/Functional/SmokeTests/smoke_test_utils.py b/Testing/Functional/SmokeTests/smoke_test_utils.py new file mode 100644 index 00000000..7073750c --- /dev/null +++ b/Testing/Functional/SmokeTests/smoke_test_utils.py @@ -0,0 +1,268 @@ +""" +Helper methods for running the functional smoke tests. +""" + +import os +import json +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions +from scubagoggles.orchestrator import Orchestrator +from scubagoggles.utils import get_package_version + +OUTPUT_DIRECTORY = "GWSBaselineConformance" +BASELINE_REPORT_H1 = "SCuBA GWS Security Baseline Conformance Reports" +CISA_GOV_URL = "https://www.cisa.gov/scuba" +SCUBAGOGGLES_BASELINES_URL = "https://github.com/cisagov/ScubaGoggles/tree/main/baselines" + +def get_output_path() -> str: + """ + Get the latest output directory created by `scubagoggles gws`. + The default name is "GWSBaselineConformance_. + + Returns: + str: The path to the latest output directory + """ + directories: list = [ + d for d in os.listdir() + if os.path.isdir(d) and d.startswith(OUTPUT_DIRECTORY) + ] + directories.sort(key=os.path.getctime, reverse=True) + return os.path.join(os.getcwd(), directories[0]) + +def prepend_file_protocol(path: str) -> str: + """ + Prepends "file://", which is used to locate files on a local filesystem. + + Returns: + str: Path to a file with the local filesystem prepended + """ + if not path.startswith("file://"): + path = "file://" + path + return path + +def verify_output_type(output_path: str, output: list) -> list: + """ + Checks if the output generated from `scubagoggles` creates the correct output. + Validate files/directories and catch invalid json. + + Args: + output_path: The output path, i.e. "GWSBaselineConformance_" + output: Initialized as an empty list + + Returns: + list: All output file and directory names + """ + entries: list = os.listdir(output_path) + for entry in entries: + output.append(entry) + + # Check if entry is a valid directory or file + # If a valid directory, then recurse + child_path: str = os.path.join(output_path, entry) + if os.path.isdir(child_path): + assert True + verify_output_type(child_path, output) + elif os.path.isfile(child_path): + + # Check for valid json + if child_path.endswith(".json"): + try: + with open(child_path, encoding="utf-8") as jsonfile: + json.load(jsonfile) + except ValueError as e: + raise ValueError(f"{child_path} contains invalid json") from e + assert True + else: + raise OSError("Entry is not a directory or file (symlink, etc.)") + return output + +def get_required_entries(sample_report, required_entries) -> list: + """ + From the "sample-report" directory, add all file and directory names + into a list "required_entries". All entries must be present + for smoke tests to pass. + + Args: + sample_report: Path where "sample-report" is located in the project + required_entries: Initialized as an empty list + + Returns: + list: All required file and directory names + """ + with os.scandir(sample_report) as entries: + for entry in entries: + required_entries.append(entry.name) + if entry.is_dir(): + get_required_entries(entry.path, required_entries) + return required_entries + +def verify_all_outputs_exist(output: list, required_entries: list): + """ + Verify all files and directories are created after running `scubagoggles gws`. + + Args: + output: a list of all files and directories generated by `scubagoggles gws` + required_entries: a list of all required file and directory names + """ + for required_entry in required_entries: + if required_entry in output: + assert True + else: + raise ValueError(f"{required_entry} was not found in the generated report") + +def verify_scubaresults(jsonfile): + """ + Verify "ScubaResults.json" is valid, and check if any errors + are displayed in the reports. + + Args: + jsonfile: Path to a json file + """ + scubaresults = json.load(jsonfile) + summaries = scubaresults["Summary"] + for product, summary in summaries.items(): + if summary["Errors"] != 0: + raise ValueError(f"{product} contains errors in the report") + +def run_selenium(browser, domain): + """ + Run Selenium tests against the generated reports. + + Args: + browser: A Selenium WebDriver instance + domain: The user's domain + """ + verify_navigation_links(browser) + h1 = browser.find_element(By.TAG_NAME, "h1").text + assert h1 == BASELINE_REPORT_H1 + + gws_products = Orchestrator.gws_products() + products = { + product: { "title": f"{product} Baseline Report" } + for product in gws_products["prod_to_fullname"].values() + } + + # Before entering loop check that we actually display 10 rows in table + reports_table = get_reports_table(browser) + + if len(reports_table) == 10: + for i in range(len(reports_table)): + + # Check if domain is present in agency table + # Skip tool version if assessing the parent report + verify_tenant_table(browser, domain, True) + + reports_table = get_reports_table(browser)[i] + baseline_report = reports_table.find_elements(By.TAG_NAME, "td")[0] + product = baseline_report.text + assert product in products + + individual_report_anchor = baseline_report.find_element(By.TAG_NAME, "a") + individual_report_anchor_href = individual_report_anchor.get_attribute("href") + individual_report_anchor.click() + current_url = browser.current_url() + assert individual_report_anchor_href == current_url + + # Check at the individual report level + verify_navigation_links(browser) + h1 = browser.find_element(By.TAG_NAME, "h1").text + assert h1 == products[product]["title"] + + # Check if domain and tool version are present in individual report + verify_tenant_table(browser, domain, False) + + policy_tables = browser.find_elements(By.TAG_NAME, "table") + for table in policy_tables[1:]: + + # Verify policy table headers are correct + headers = ( + table.find_element(By.TAG_NAME, "thead") + .find_elements(By.TAG_NAME, "tr")[0] + .find_elements(By.TAG_NAME, "th") + ) + assert len(headers) == 5 + assert headers[0].text == "Control ID" + assert headers[1].text in "Requirements" or headers[1].text in "Rule Name" + assert headers[2].text == "Result" + assert headers[3].text == "Criticality" + assert headers[4].text in "Details" or headers[4].text in "Rule Description" + + # Verify policy table rows are populated + tbody = table.find_element(By.TAG_NAME, "tbody") + rows = tbody.find_elements(By.TAG_NAME, "tr") + assert len(rows) > 0 + + parent_report_anchor = ( + browser.find_element(By.TAG_NAME, "header") + .find_element(By.TAG_NAME, "a") + ) + parent_report_anchor_href = parent_report_anchor.get_attribute("href") + parent_report_anchor.click() + current_url = browser.current_url() + assert parent_report_anchor_href == current_url + + WebDriverWait(browser, 10).until( + expected_conditions.presence_of_element_located( + (By.TAG_NAME, "body") + ) + ) + else: + raise ValueError("Expected the reports table to have a length of 10") + +def verify_navigation_links(browser): + """ + For each baseline report, check that the navigation links display correctly. + + Args: + browser: A Selenium WebDriver instance + """ + links = ( + browser.find_element(By.CLASS_NAME, "links") + .find_elements(By.TAG_NAME, "a") + ) + if len(links) == 2: + assert links[0].get_attribute("href") == CISA_GOV_URL + assert links[1].get_attribute("href") == SCUBAGOGGLES_BASELINES_URL + +def get_reports_table(browser): + """ + Get the reports table element from the DOM. + (Table in BaselineReports.html with list of baselines and pass/fail/warning of each) + + Args: + browser: A Selenium WebDriver instance + """ + return ( + browser.find_elements(By.TAG_NAME, "table")[1] + .find_element(By.TAG_NAME, "tbody") + .find_elements(By.TAG_NAME, "tr") + ) + +def verify_tenant_table(browser, domain, parent): + """ + Get the tenant table rows elements from the DOM. + (Table at the top of each report with user domain, baseline/tool version) + + Args: + browser: A Selenium WebDriver instance + domain: The user's domain + parent: boolean to determine parent/individual reports + """ + tenant_table_rows = ( + browser.find_element(By.TAG_NAME, "table") + .find_element(By.TAG_NAME, "tbody") + .find_elements(By.TAG_NAME, "tr") + ) + assert len(tenant_table_rows) == 2 + customer_domain = tenant_table_rows[1].find_elements(By.TAG_NAME, "td")[0].text + assert customer_domain == domain + + if not parent: + # Check for correct tool version, e.g. 0.2.0 + version = get_package_version("scubagoggles") + tool_version = tenant_table_rows[1].find_elements(By.TAG_NAME, "td")[3].text + assert version == tool_version + + # Baseline version should also be checked in this method + # Add as an additional todo diff --git a/Testing/Functional/conftest.py b/Testing/Functional/conftest.py new file mode 100644 index 00000000..456f9be3 --- /dev/null +++ b/Testing/Functional/conftest.py @@ -0,0 +1,45 @@ +""" + conftest.py serves as a configuration file for pytest. +""" + +import pytest +from SmokeTests.selenium_browser import Browser + +def pytest_addoption(parser): + """ + Add custom cli arguments when running `pytest`. + + Args: + parser: An instance of "argparse.ArgumentParser" + """ + parser.addoption("--subjectemail", action="store") + parser.addoption("--domain", action="store") + +@pytest.fixture +def subjectemail(pytestconfig): + """ + Setup code that shares the "subjectemail" parameter across tests. + + Args: + pytestconfig: Provides access to the "Config" object for a current test session + """ + return pytestconfig.getoption("subjectemail") + +@pytest.fixture +def domain(pytestconfig): + """ + Setup code that shares the "domain" parameter across tests. + + Args: + pytestconfig: Provides access to the "Config" object for a current test session + """ + return pytestconfig.getoption("domain") + +@pytest.fixture +def browser(): + """ + Setup code that shares a Selenium WebDriver instance across tests. + """ + browser_instance = Browser() + yield browser_instance + browser_instance.quit() diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..5947a673 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +python_classes = *Test +pythonpath = . scubagoggles \ No newline at end of file diff --git a/scubagoggles/utils.py b/scubagoggles/utils.py index 41b70501..ae45a454 100644 --- a/scubagoggles/utils.py +++ b/scubagoggles/utils.py @@ -4,6 +4,7 @@ """ from pathlib import Path +from importlib.metadata import version, PackageNotFoundError def create_subset_inverted_dict(dictionary: dict, keys: list) -> dict: """ @@ -53,3 +54,13 @@ def rel_abs_path(file_path: str, rel_path) -> str: """ current_dir = Path(file_path).resolve().parent return (current_dir / rel_path).resolve() + +def get_package_version(package: str) -> str: + """ + Get the current version for a package + """ + try: + package_version = version(package) + return package_version + except PackageNotFoundError as e: + raise PackageNotFoundError("Package was not found") from e