Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
address final pylinter errors, add get_package_version() method in or…
Browse files Browse the repository at this point in the history
…chestrator/utils
mitchelbaker-cisa committed Aug 6, 2024
1 parent e449b5c commit 0351ca8
Showing 5 changed files with 95 additions and 67 deletions.
3 changes: 1 addition & 2 deletions Testing/Functional/SmokeTests/selenium_browser.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
"""
selenium_browser.py declares a Browser class for use in ScubaGoggles testing.
selenium_browser.py declares a Browser class for use in ScubaGoggles testing.
"""

from selenium import webdriver
@@ -10,7 +10,6 @@ class Browser:
The Browser class encapsulates the setup, usage, and teardown of a
Selenium WebDriver instance for automated browser interactions.
"""

def __init__(self):
chrome_options = Options()
chrome_options.add_argument("--headless")
18 changes: 9 additions & 9 deletions Testing/Functional/SmokeTests/smoke_test.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
"""
smoke_test.py declares a SmokeTest class for ScubaGoggles automation testing.
smoke_test.py declares a SmokeTest class for ScubaGoggles automation testing.
"""

import subprocess
@@ -30,10 +30,10 @@ class SmokeTest:
"""
def test_scubagoggles_output(self, subjectemail):
"""
Test if the `scubagoggles gws` command generates correct output for all baselines.
Args:
subjectemail: The email address of a user for the service account
Test if the `scubagoggles gws` command generates correct output for all baselines.
Args:
subjectemail: The email address of a user for the service account
"""
try:
command: str = f"scubagoggles gws --subjectemail {subjectemail} --quiet"
@@ -47,7 +47,7 @@ def test_scubagoggles_output(self, subjectemail):

def test_scubaresults(self):
"""
Determine if ScubaResults.json contains API errors or exceptions.
Determine if ScubaResults.json contains API errors or exceptions.
"""
try:
output_path: str = get_output_path()
@@ -59,14 +59,14 @@ def test_scubaresults(self):

def test_scubagoggles_report(self, browser, domain):
"""
Test if the generated baseline reports are correct,
i.e. BaselineReports.html, CalendarReport.html, ChatReport.html
Test if the generated baseline reports are correct,
i.e. BaselineReports.html, CalendarReport.html, ChatReport.html
"""
try:
output_path: str = get_output_path()
report_path: str = prepend_file_protocol(os.path.join(output_path, BASELINE_REPORTS))
browser.get(report_path)
run_selenium(browser, domain)
except (ValueError, Exception) as e:
except (ValueError, AssertionError, Exception) as e:
browser.quit()
pytest.fail(f"An error occurred, {e}")
126 changes: 70 additions & 56 deletions Testing/Functional/SmokeTests/smoke_test_utils.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
"""
Helper methods for running the functional smoke tests.
Helper methods for running the functional smoke tests.
"""

import os
@@ -8,6 +8,7 @@
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from scubagoggles.orchestrator import Orchestrator
from scubagoggles.utils import get_package_version

OUTPUT_DIRECTORY = "GWSBaselineConformance"
BASELINE_REPORT_H1 = "SCuBA GWS Security Baseline Conformance Reports"
@@ -16,41 +17,41 @@

def get_output_path() -> str:
"""
Get the latest output directory created by `scubagoggles gws`.
The default name is "GWSBaselineConformance_<timestamp>.
Get the latest output directory created by `scubagoggles gws`.
The default name is "GWSBaselineConformance_<timestamp>.
Returns:
str: The path to the latest output directory
Returns:
str: The path to the latest output directory
"""
directories: list = [
d for d in os.listdir()
if os.path.isdir(d) and d.startswith(OUTPUT_DIRECTORY)
]
directories.sort(key=lambda d: os.path.getctime(d), reverse=True)
directories.sort(key=os.path.getctime, reverse=True)
return os.path.join(os.getcwd(), directories[0])

def prepend_file_protocol(path: str) -> str:
"""
Prepends "file://", which is used to locate files on a local filesystem.
Prepends "file://", which is used to locate files on a local filesystem.
Returns:
str: Path to a file with the local filesystem prepended
Returns:
str: Path to a file with the local filesystem prepended
"""
if not path.startswith("file://"):
path = "file://" + path
return path

def verify_output_type(output_path: str, output: list) -> list:
"""
Checks if the output generated from `scubagoggles` creates the correct output.
Validate files/directories and catch invalid json.
Checks if the output generated from `scubagoggles` creates the correct output.
Validate files/directories and catch invalid json.
Args:
output_path: The output path, i.e. "GWSBaselineConformance_<timestamp>"
output: Initialized as an empty list
Args:
output_path: The output path, i.e. "GWSBaselineConformance_<timestamp>"
output: Initialized as an empty list
Returns:
list: All output file and directory names
Returns:
list: All output file and directory names
"""
entries: list = os.listdir(output_path)
for entry in entries:
@@ -78,16 +79,16 @@ def verify_output_type(output_path: str, output: list) -> list:

def get_required_entries(sample_report, required_entries) -> list:
"""
From the "sample-report" directory, add all file and directory names
into a list "required_entries". All entries must be present
for smoke tests to pass.
From the "sample-report" directory, add all file and directory names
into a list "required_entries". All entries must be present
for smoke tests to pass.
Args:
sample_report: Path where "sample-report" is located in the project
required_entries: Initialized as an empty list
Args:
sample_report: Path where "sample-report" is located in the project
required_entries: Initialized as an empty list
Returns:
list: All required file and directory names
Returns:
list: All required file and directory names
"""
with os.scandir(sample_report) as entries:
for entry in entries:
@@ -98,11 +99,11 @@ def get_required_entries(sample_report, required_entries) -> list:

def verify_all_outputs_exist(output: list, required_entries: list):
"""
Verify all files and directories are created after running `scubagoggles gws`.
Verify all files and directories are created after running `scubagoggles gws`.
Args:
output: a list of all files and directories generated by `scubagoggles gws`
required_entries: a list of all required file and directory names
Args:
output: a list of all files and directories generated by `scubagoggles gws`
required_entries: a list of all required file and directory names
"""
for required_entry in required_entries:
if required_entry in output:
@@ -112,11 +113,11 @@ def verify_all_outputs_exist(output: list, required_entries: list):

def verify_scubaresults(jsonfile):
"""
Verify "ScubaResults.json" is valid, and check if any errors
are displayed in the reports.
Verify "ScubaResults.json" is valid, and check if any errors
are displayed in the reports.
Args:
jsonfile: Path to a json file
Args:
jsonfile: Path to a json file
"""
scubaresults = json.load(jsonfile)
summaries = scubaresults["Summary"]
@@ -126,11 +127,11 @@ def verify_scubaresults(jsonfile):

def run_selenium(browser, domain):
"""
Run Selenium tests against the generated reports.
Run Selenium tests against the generated reports.
Args:
browser: A Selenium WebDriver instance
domain: The user's domain
Args:
browser: A Selenium WebDriver instance
domain: The user's domain
"""
verify_navigation_links(browser)
h1 = browser.find_element(By.TAG_NAME, "h1").text
@@ -149,10 +150,8 @@ def run_selenium(browser, domain):
for i in range(len(reports_table)):

# Check if domain is present in agency table
verify_tenant_table(browser, domain)

# TODO
# Check for correct baseline and tool version, e.g. 0.2, 0.2.0
# Skip tool version if assessing the parent report
verify_tenant_table(browser, domain, True)

reports_table = get_reports_table(browser)[i]
baseline_report = reports_table.find_elements(By.TAG_NAME, "td")[0]
@@ -170,7 +169,8 @@ def run_selenium(browser, domain):
h1 = browser.find_element(By.TAG_NAME, "h1").text
assert h1 == products[product]["title"]

verify_tenant_table(browser, domain)
# Check if domain and tool version are present in individual report
verify_tenant_table(browser, domain, False)

policy_tables = browser.find_elements(By.TAG_NAME, "table")
for table in policy_tables[1:]:
@@ -183,10 +183,10 @@ def run_selenium(browser, domain):
)
assert len(headers) == 5
assert headers[0].text == "Control ID"
assert headers[1].text in { "Requirement", "Rule Name" }
assert headers[1].text in "Requirements" or headers[1].text in "Rule Name"
assert headers[2].text == "Result"
assert headers[3].text == "Criticality"
assert headers[4].text == { "Details", "Rule Description" }
assert headers[4].text in "Details" or headers[4].text in "Rule Description"

# Verify policy table rows are populated
tbody = table.find_element(By.TAG_NAME, "tbody")
@@ -212,10 +212,10 @@ def run_selenium(browser, domain):

def verify_navigation_links(browser):
"""
For each baseline report, check that the navigation links display correctly.
For each baseline report, check that the navigation links display correctly.
Args:
browser: A Selenium WebDriver instance
Args:
browser: A Selenium WebDriver instance
"""
links = (
browser.find_element(By.CLASS_NAME, "links")
@@ -227,28 +227,42 @@ def verify_navigation_links(browser):

def get_reports_table(browser):
"""
Get the reports table element from the DOM.
(Table in BaselineReports.html with list of baselines and pass/fail/warning of each)
Get the reports table element from the DOM.
(Table in BaselineReports.html with list of baselines and pass/fail/warning of each)
Args:
browser: A Selenium WebDriver instance
Args:
browser: A Selenium WebDriver instance
"""
return (
browser.find_elements(By.TAG_NAME, "table")[1]
.find_element(By.TAG_NAME, "tbody")
.find_elements(By.TAG_NAME, "tr")
)

def verify_tenant_table(browser, domain):
def verify_tenant_table(browser, domain, parent):
"""
Get the tenant table element from the DOM.
(Table at the top of each report with user domain, baseline/tool version)
Get the tenant table rows elements from the DOM.
(Table at the top of each report with user domain, baseline/tool version)
Args:
browser: A Selenium WebDriver instance
domain: The user's domain
parent: boolean to determine parent/individual reports
"""
tenant_table = (
tenant_table_rows = (
browser.find_element(By.TAG_NAME, "table")
.find_element(By.TAG_NAME, "tbody")
.find_elements(By.TAG_NAME, "tr")
)
assert len(tenant_table) == 2
customer_domain = tenant_table[1].find_elements(By.TAG_NAME, "td")[0].text
assert len(tenant_table_rows) == 2
customer_domain = tenant_table_rows[1].find_elements(By.TAG_NAME, "td")[0].text
assert customer_domain == domain

if not parent:
# Check for correct tool version, e.g. 0.2.0
version = get_package_version("scubagoggles")
tool_version = tenant_table_rows[1].find_elements(By.TAG_NAME, "td")[3].text
assert version == tool_version

# Baseline version should also be checked in this method
# Add as an additional todo
4 changes: 4 additions & 0 deletions Testing/Functional/conftest.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
"""
conftest.py serves as a configuration file for pytest.
"""

import pytest
from SmokeTests.selenium_browser import Browser

11 changes: 11 additions & 0 deletions scubagoggles/utils.py
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@
"""

from pathlib import Path
from importlib.metadata import version, PackageNotFoundError

def create_subset_inverted_dict(dictionary: dict, keys: list) -> dict:
"""
@@ -53,3 +54,13 @@ def rel_abs_path(file_path: str, rel_path) -> str:
"""
current_dir = Path(file_path).resolve().parent
return (current_dir / rel_path).resolve()

def get_package_version(package: str) -> str:
"""
Get the current version for a package
"""
try:
package_version = version(package)
return package_version
except PackageNotFoundError as e:
raise PackageNotFoundError(f"Package was not found, {e}")

0 comments on commit 0351ca8

Please sign in to comment.