From 59468fd5db925f68e07e00e5b5354e83e0c6f0b1 Mon Sep 17 00:00:00 2001 From: Alden Hilton Date: Tue, 23 Jan 2024 15:58:14 -0800 Subject: [PATCH 1/7] Implement warning for missing output --- scubagoggles/orchestrator.py | 10 ++- scubagoggles/reporter/reporter.py | 103 +++++++++++++++----------- scubagoggles/reporter/scripts/main.js | 4 +- 3 files changed, 71 insertions(+), 46 deletions(-) diff --git a/scubagoggles/orchestrator.py b/scubagoggles/orchestrator.py index be3d4188..e65860ae 100644 --- a/scubagoggles/orchestrator.py +++ b/scubagoggles/orchestrator.py @@ -208,6 +208,7 @@ def run_reporter(args): n_warn = stats["Warning"] n_fail = stats["Fail"] n_manual = stats["N/A"] + stats["No events found"] + n_error = stats["Error"] pass_summary = (f"
{n_success}" f" {pluralize('test', 'tests', n_success)} passed
") @@ -218,6 +219,7 @@ def run_reporter(args): warning_summary = "
" failure_summary = "
" manual_summary = "
" + error_summary = "
" if n_warn > 0: warning_summary = (f"
{n_warn}" @@ -228,10 +230,14 @@ def run_reporter(args): if n_manual > 0: manual_summary = (f"
{n_manual} manual" f" {pluralize('check', 'checks', n_manual)} needed
") + if n_error > 0: + error_summary = (f"
{n_error}" + f" {pluralize('error', 'errors', n_error)}
") table_data.append({ - "Baseline Conformance Reports": link, - "Details": f"{pass_summary}{warning_summary}{failure_summary}{manual_summary}" + "Baseline Conformance Reports": link, + "Details": f"{pass_summary}{warning_summary}{failure_summary}{manual_summary}\ + {error_summary}" }) fragments.append(reporter.create_html_table(table_data)) diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py index 57065b7f..564c0f75 100644 --- a/scubagoggles/reporter/reporter.py +++ b/scubagoggles/reporter/reporter.py @@ -5,10 +5,13 @@ """ import os import time +import warnings from datetime import datetime import pandas as pd from scubagoggles.utils import rel_abs_path +SCUBA_GITHUB_URL = "https://github.com/cisagov/scubagoggles" + def get_test_result(requirement_met : bool, criticality : str, no_such_events : bool) -> str: ''' Checks the Rego to see if the baseline passed or failed and indicates the criticality @@ -164,56 +167,72 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str, "Warning": 0, "Fail": 0, "N/A": 0, - "No events found": 0 + "No events found": 0, + "Error": 0 } for baseline_group in product_policies: table_data = [] for control in baseline_group['Controls']: tests = [test for test in test_results_data if test['PolicyId'] == control['Id']] - for test in tests: - result = get_test_result(test['RequirementMet'], test['Criticality'], - test['NoSuchEvent']) - report_stats[result] = report_stats[result] + 1 - details = test['ReportDetails'] - - if result == "No events found": - warning_icon = "\ - " - details = warning_icon + " " + test['ReportDetails'] - - # As rules doesn't have it's own baseline, Rules and Common Controls - # need to be handled specially - if product_capitalized == "Rules": - if 'Not-Implemented' in test['Criticality']: - # The easiest way to identify the GWS.COMMONCONTROLS.14.1v1 - # results that belong to the Common Controls report is they're - # marked as Not-Implemented. This if excludes them from the - # rules report. - continue - table_data.append({ - 'Control ID': control['Id'], - 'Rule Name': test['Requirement'], - 'Result': result, - 'Criticality': test['Criticality'], - 'Rule Description': test['ReportDetails']}) - elif product_capitalized == "Commoncontrols" \ - and baseline_group['GroupName'] == 'System-defined Rules' \ - and 'Not-Implemented' not in test['Criticality']: - # The easiest way to identify the System-defined Rules - # results that belong to the Common Controls report is they're - # marked as Not-Implemented. This if excludes the full results - # from the Common Controls report. - continue - else: - table_data.append({ + if len(tests) == 0: + # Handle the case where Rego doesn't output anything for a given control + report_stats['Error'] += 1 + issues_link = f'GitHub' + table_data.append({ 'Control ID': control['Id'], 'Requirement': control['Value'], - 'Result': result, - 'Criticality': test['Criticality'], - 'Details': details}) + 'Result': "Error - Test results missing", + 'Criticality': "-", + 'Details': f'Report issue on {issues_link}' + }) + warnings.warn(f"No test results found for Control Id {control['Id']}", + RuntimeWarning) + else: + for test in tests: + result = get_test_result(test['RequirementMet'], test['Criticality'], + test['NoSuchEvent']) + report_stats[result] = report_stats[result] + 1 + details = test['ReportDetails'] + + if result == "No events found": + warning_icon = "\ + " + details = warning_icon + " " + test['ReportDetails'] + + # As rules doesn't have its own baseline, Rules and Common Controls + # need to be handled specially + if product_capitalized == "Rules": + if 'Not-Implemented' in test['Criticality']: + # The easiest way to identify the GWS.COMMONCONTROLS.13.1v1 + # results that belong to the Common Controls report is they're + # marked as Not-Implemented. This if excludes them from the + # rules report. + continue + table_data.append({ + 'Control ID': control['Id'], + 'Rule Name': test['Requirement'], + 'Result': result, + 'Criticality': test['Criticality'], + 'Rule Description': test['ReportDetails']}) + elif product_capitalized == "Commoncontrols" \ + and baseline_group['GroupName'] == 'System-defined Rules' \ + and 'Not-Implemented' not in test['Criticality']: + # The easiest way to identify the System-defined Rules + # results that belong to the Common Controls report is they're + # marked as Not-Implemented. This if excludes the full results + # from the Common Controls report. + continue + else: + table_data.append({ + 'Control ID': control['Id'], + 'Requirement': control['Value'], + 'Result': result, + 'Criticality': test['Criticality'], + 'Details': details + }) fragments.append(f"

{product_upper}-{baseline_group['GroupNumber']} \ {baseline_group['GroupName']}

") fragments.append(create_html_table(table_data)) diff --git a/scubagoggles/reporter/scripts/main.js b/scubagoggles/reporter/scripts/main.js index d03f19a6..b48881f8 100644 --- a/scubagoggles/reporter/scripts/main.js +++ b/scubagoggles/reporter/scripts/main.js @@ -32,8 +32,8 @@ const colorRows = () => { } else if (rows[i].children[statusCol].innerHTML.includes("Error")) { rows[i].style.background = "var(--test-fail)"; - rows[i].querySelectorAll('td')[1].style.borderColor = "var(--border-color)"; - rows[i].querySelectorAll('td')[1].style.color = "#d10000"; + rows[i].querySelectorAll('td')[statusCol].style.borderColor = "var(--border-color)"; + rows[i].querySelectorAll('td')[statusCol].style.color = "#d10000"; } } catch (error) { From d2c5fe6cd9c282b4fbce7a1769eed628f64ad0cd Mon Sep 17 00:00:00 2001 From: Alden Hilton Date: Tue, 23 Jan 2024 16:09:18 -0800 Subject: [PATCH 2/7] Add missing reo spf check --- Testing/RegoTests/gmail/gmail03_test.rego | 30 ++++++++++++++++++++--- rego/Gmail.rego | 15 +++++++++++- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/Testing/RegoTests/gmail/gmail03_test.rego b/Testing/RegoTests/gmail/gmail03_test.rego index 199c8ac6..4cd04437 100644 --- a/Testing/RegoTests/gmail/gmail03_test.rego +++ b/Testing/RegoTests/gmail/gmail03_test.rego @@ -5,9 +5,31 @@ import future.keywords # # GWS.GMAIL.3.1v0.1 #-- +test_MaintainList_Correct_V1 if { + # Test not implemented + PolicyId := "GWS.GMAIL.17.1v0.1" + Output := tests with input as { + "gmail_logs": {"items": [ + ]}, + "tenant_info": { + "topLevelOU": "" + } + } + + RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId] + count(RuleOutput) == 1 + not RuleOutput[0].RequirementMet + not RuleOutput[0].NoSuchEvent + RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check." +} +#-- + +# +# GWS.GMAIL.3.2v0.1 +#-- test_SPF_Correct_V1 if { # Test SPF when there's only one domain - PolicyId := "GWS.GMAIL.3.1v0.1" + PolicyId := "GWS.GMAIL.3.2v0.1" Output := tests with input as { "dkim_records": [ { @@ -32,7 +54,7 @@ test_SPF_Correct_V1 if { test_SPF_Correct_V2 if { # Test SPF when there's multiple domains - PolicyId := "GWS.GMAIL.3.1v0.1" + PolicyId := "GWS.GMAIL.3.2v0.1" Output := tests with input as { "dkim_records": [ { @@ -65,7 +87,7 @@ test_SPF_Correct_V2 if { test_SPF_Incorrect_V1 if { # Test SPF when there's multiple domains and only one is correct - PolicyId := "GWS.GMAIL.3.1v0.1" + PolicyId := "GWS.GMAIL.3.2v0.1" Output := tests with input as { "dkim_records": [ { @@ -98,7 +120,7 @@ test_SPF_Incorrect_V1 if { test_SPF_Incorrect_V2 if { # Test SPF when there's only one domain and it's wrong - PolicyId := "GWS.GMAIL.3.1v0.1" + PolicyId := "GWS.GMAIL.3.2v0.1" Output := tests with input as { "dkim_records": [ { diff --git a/rego/Gmail.rego b/rego/Gmail.rego index b7a93419..238d6249 100644 --- a/rego/Gmail.rego +++ b/rego/Gmail.rego @@ -107,6 +107,19 @@ if { # # Baseline GWS.GMAIL.3.1v0.1 #-- +# No implementation steps provided for this policy +tests contains { + "PolicyId": "GWS.GMAIL.3.1v0.1", + "Criticality": "Shall/Not-Implemented", + "ReportDetails": "Currently not able to be tested automatically; please manually check.", + "ActualValue": "", + "RequirementMet": false, + "NoSuchEvent": false} +#-- + +# +# Baseline GWS.GMAIL.3.2v0.1 +#-- DomainsWithSpf contains SpfRecord.domain if { some SpfRecord in input.spf_records some Rdata in SpfRecord.rdata @@ -114,7 +127,7 @@ DomainsWithSpf contains SpfRecord.domain if { } tests contains { - "PolicyId": "GWS.GMAIL.3.1v0.1", + "PolicyId": "GWS.GMAIL.3.2v0.1", "Criticality": "Shall", "ReportDetails": ReportDetailsArray(Status, DomainsWithoutSpf, AllDomains), "ActualValue": DomainsWithoutSpf, From 74ce7675ccdde9ae58811bb8b362f12372dd1c84 Mon Sep 17 00:00:00 2001 From: Alden Hilton Date: Tue, 23 Jan 2024 16:13:48 -0800 Subject: [PATCH 3/7] Add rego check for 10.2 --- Testing/RegoTests/gmail/gmail10_test.rego | 20 ++++++++++++++++++++ rego/Gmail.rego | 12 ++++++++++++ 2 files changed, 32 insertions(+) diff --git a/Testing/RegoTests/gmail/gmail10_test.rego b/Testing/RegoTests/gmail/gmail10_test.rego index 00a3f2b7..3609a40b 100644 --- a/Testing/RegoTests/gmail/gmail10_test.rego +++ b/Testing/RegoTests/gmail/gmail10_test.rego @@ -310,5 +310,25 @@ test_GoogleWorkspaceSync_Incorrect_V5 if { not RuleOutput[0].NoSuchEvent RuleOutput[0].ReportDetails == "Requirement failed in Secondary OU." } +#-- +# +# GWS.GMAIL.10.2v0.1 +test_May_Correct_V1 if { + # Test Comprehensive Mail Storage when there's only one event + PolicyId := "GWS.GMAIL.10.2v0.1" + Output := tests with input as { + "gmail_logs": {"items": [ + ]}, + "tenant_info": { + "topLevelOU": "" + } + } + + RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId] + count(RuleOutput) == 1 + not RuleOutput[0].RequirementMet + not RuleOutput[0].NoSuchEvent + RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check." +} #-- \ No newline at end of file diff --git a/rego/Gmail.rego b/rego/Gmail.rego index 238d6249..681f289f 100644 --- a/rego/Gmail.rego +++ b/rego/Gmail.rego @@ -1362,6 +1362,18 @@ if { } #-- +# +# Baseline GWS.GMAIL.10.2v0.1 +#-- +# No implementation steps provided for this policy +tests contains { + "PolicyId": "GWS.GMAIL.10.2v0.1", + "Criticality": "May/Not-Implemented", + "ReportDetails": "Currently not able to be tested automatically; please manually check.", + "ActualValue": "", + "RequirementMet": false, + "NoSuchEvent": false} +#-- ################ # GWS.GMAIL.11 # From 33ad8979e3d855f2e6a856cd9dc6a8a66ebdd86d Mon Sep 17 00:00:00 2001 From: Alden Hilton Date: Tue, 23 Jan 2024 16:17:48 -0800 Subject: [PATCH 4/7] Add rego check for 15.2 --- Testing/RegoTests/gmail/gmail10_test.rego | 3 ++- Testing/RegoTests/gmail/gmail15_test.rego | 22 ++++++++++++++++++++++ rego/Gmail.rego | 22 +++++++++++++++++++--- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/Testing/RegoTests/gmail/gmail10_test.rego b/Testing/RegoTests/gmail/gmail10_test.rego index 3609a40b..8e0aa7d6 100644 --- a/Testing/RegoTests/gmail/gmail10_test.rego +++ b/Testing/RegoTests/gmail/gmail10_test.rego @@ -314,8 +314,9 @@ test_GoogleWorkspaceSync_Incorrect_V5 if { # # GWS.GMAIL.10.2v0.1 +#-- test_May_Correct_V1 if { - # Test Comprehensive Mail Storage when there's only one event + # Test not implemented PolicyId := "GWS.GMAIL.10.2v0.1" Output := tests with input as { "gmail_logs": {"items": [ diff --git a/Testing/RegoTests/gmail/gmail15_test.rego b/Testing/RegoTests/gmail/gmail15_test.rego index 14ca6145..eadd92dc 100644 --- a/Testing/RegoTests/gmail/gmail15_test.rego +++ b/Testing/RegoTests/gmail/gmail15_test.rego @@ -348,4 +348,26 @@ test_EnhancedPreDeliveryMessageScanning_Incorrect_V5 if { not RuleOutput[0].NoSuchEvent RuleOutput[0].ReportDetails == "Requirement failed in Secondary OU." } +#-- + +# +# GWS.GMAIL.15.2v0.1 +#-- +test_Other_Correct_V1 if { + # Test not implemented + PolicyId := "GWS.GMAIL.15.2v0.1" + Output := tests with input as { + "gmail_logs": {"items": [ + ]}, + "tenant_info": { + "topLevelOU": "" + } + } + + RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId] + count(RuleOutput) == 1 + not RuleOutput[0].RequirementMet + not RuleOutput[0].NoSuchEvent + RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check." +} #-- \ No newline at end of file diff --git a/rego/Gmail.rego b/rego/Gmail.rego index 681f289f..5c845e1c 100644 --- a/rego/Gmail.rego +++ b/rego/Gmail.rego @@ -114,7 +114,8 @@ tests contains { "ReportDetails": "Currently not able to be tested automatically; please manually check.", "ActualValue": "", "RequirementMet": false, - "NoSuchEvent": false} + "NoSuchEvent": false +} #-- # @@ -530,7 +531,8 @@ tests contains { "ReportDetails": "Currently not able to be tested automatically; please manually check.", "ActualValue": "", "RequirementMet": false, - "NoSuchEvent": false} + "NoSuchEvent": false +} #-- ############### @@ -1372,7 +1374,8 @@ tests contains { "ReportDetails": "Currently not able to be tested automatically; please manually check.", "ActualValue": "", "RequirementMet": false, - "NoSuchEvent": false} + "NoSuchEvent": false +} #-- ################ @@ -1633,6 +1636,19 @@ if { } #-- +# +# Baseline GWS.GMAIL.15.2v0.1 +#-- +# No implementation steps provided for this policy +tests contains { + "PolicyId": "GWS.GMAIL.15.2v0.1", + "Criticality": "Should/Not-Implemented", + "ReportDetails": "Currently not able to be tested automatically; please manually check.", + "ActualValue": "", + "RequirementMet": false, + "NoSuchEvent": false +} +#-- ################ # GWS.GMAIL.16 # From 09d6e05b72293ea9b40d4d200745a2b67c400567 Mon Sep 17 00:00:00 2001 From: Alden Hilton Date: Tue, 23 Jan 2024 16:19:28 -0800 Subject: [PATCH 5/7] Add rego check for 16.2 --- Testing/RegoTests/gmail/gmail16_test.rego | 22 ++++++++++++++++++++++ rego/Gmail.rego | 13 +++++++++++++ 2 files changed, 35 insertions(+) diff --git a/Testing/RegoTests/gmail/gmail16_test.rego b/Testing/RegoTests/gmail/gmail16_test.rego index fdda42ab..e0bf7a02 100644 --- a/Testing/RegoTests/gmail/gmail16_test.rego +++ b/Testing/RegoTests/gmail/gmail16_test.rego @@ -310,4 +310,26 @@ test_SecuritySandbox_Incorrect_V5 if { not RuleOutput[0].NoSuchEvent RuleOutput[0].ReportDetails == "Requirement failed in Secondary OU." } +#-- + +# +# GWS.GMAIL.16.2v0.1 +#-- +test_Other_Correct_V1 if { + # Test not implemented + PolicyId := "GWS.GMAIL.16.2v0.1" + Output := tests with input as { + "gmail_logs": {"items": [ + ]}, + "tenant_info": { + "topLevelOU": "" + } + } + + RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId] + count(RuleOutput) == 1 + not RuleOutput[0].RequirementMet + not RuleOutput[0].NoSuchEvent + RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check." +} #-- \ No newline at end of file diff --git a/rego/Gmail.rego b/rego/Gmail.rego index 5c845e1c..913b18a3 100644 --- a/rego/Gmail.rego +++ b/rego/Gmail.rego @@ -1701,6 +1701,19 @@ if { } #-- +# +# Baseline GWS.GMAIL.16.2v0.1 +#-- +# No implementation steps provided for this policy +tests contains { + "PolicyId": "GWS.GMAIL.16.2v0.1", + "Criticality": "Should/Not-Implemented", + "ReportDetails": "Currently not able to be tested automatically; please manually check.", + "ActualValue": "", + "RequirementMet": false, + "NoSuchEvent": false +} +#-- ################ # GWS.GMAIL.17 # From 9156aadab6f69b5a59594d76903683c365c0f0e9 Mon Sep 17 00:00:00 2001 From: Alden Hilton Date: Tue, 23 Jan 2024 16:23:44 -0800 Subject: [PATCH 6/7] Add missing rego checks for group 18 --- Testing/RegoTests/gmail/gmail18_test.rego | 44 +++++++++++++++++++++++ rego/Gmail.rego | 27 ++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/Testing/RegoTests/gmail/gmail18_test.rego b/Testing/RegoTests/gmail/gmail18_test.rego index 72ecced6..b1f63d3d 100644 --- a/Testing/RegoTests/gmail/gmail18_test.rego +++ b/Testing/RegoTests/gmail/gmail18_test.rego @@ -22,4 +22,48 @@ test_AdvanvedEmailContentFitlering_Correct_V1 if { not RuleOutput[0].NoSuchEvent RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check." } +#-- + +# +# GWS.GMAIL.18.2v0.1 +#-- +test_Other_Correct_V1 if { + # Test not implemented + PolicyId := "GWS.GMAIL.18.2v0.1" + Output := tests with input as { + "gmail_logs": {"items": [ + ]}, + "tenant_info": { + "topLevelOU": "" + } + } + + RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId] + count(RuleOutput) == 1 + not RuleOutput[0].RequirementMet + not RuleOutput[0].NoSuchEvent + RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check." +} +#-- + +# +# GWS.GMAIL.18.3v0.1 +#-- +test_PII_Correct_V1 if { + # Test not implemented + PolicyId := "GWS.GMAIL.18.3v0.1" + Output := tests with input as { + "gmail_logs": {"items": [ + ]}, + "tenant_info": { + "topLevelOU": "" + } + } + + RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId] + count(RuleOutput) == 1 + not RuleOutput[0].RequirementMet + not RuleOutput[0].NoSuchEvent + RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check." +} #-- \ No newline at end of file diff --git a/rego/Gmail.rego b/rego/Gmail.rego index 913b18a3..63fc6f61 100644 --- a/rego/Gmail.rego +++ b/rego/Gmail.rego @@ -1754,6 +1754,33 @@ tests contains { } #-- +# +# Baseline GWS.GMAIL.18.2v0.1 +#-- +# No implementation steps provided for this policy +tests contains { + "PolicyId": "GWS.GMAIL.18.2v0.1", + "Criticality": "Should/Not-Implemented", + "ReportDetails": "Currently not able to be tested automatically; please manually check.", + "ActualValue": "", + "RequirementMet": false, + "NoSuchEvent": false +} +#-- + +# +# Baseline GWS.GMAIL.18.3v0.1 +#-- +# No implementation steps provided for this policy +tests contains { + "PolicyId": "GWS.GMAIL.18.3v0.1", + "Criticality": "Shall/Not-Implemented", + "ReportDetails": "Currently not able to be tested automatically; please manually check.", + "ActualValue": "", + "RequirementMet": false, + "NoSuchEvent": false +} +#-- ################ # GWS.GMAIL.19 # From e358c87713d15413429778854766163c0af6a84b Mon Sep 17 00:00:00 2001 From: Alden Hilton Date: Tue, 23 Jan 2024 16:40:46 -0800 Subject: [PATCH 7/7] Broke up the run_reporter function to satisfy linter --- scubagoggles/orchestrator.py | 70 +++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/scubagoggles/orchestrator.py b/scubagoggles/orchestrator.py index e65860ae..b3815b60 100644 --- a/scubagoggles/orchestrator.py +++ b/scubagoggles/orchestrator.py @@ -119,6 +119,42 @@ def pluralize(singular : str, plural : str, count : int) -> str: return singular return plural +def generate_summary(stats : dict) -> str: + """ + Craft the html-formatted summary from the stats dictionary. + """ + n_success = stats["Pass"] + n_warn = stats["Warning"] + n_fail = stats["Fail"] + n_manual = stats["N/A"] + stats["No events found"] + n_error = stats["Error"] + + pass_summary = (f"
{n_success}" + f" {pluralize('test', 'tests', n_success)} passed
") + + # The warnings, failures, and manuals are only shown if they are + # greater than zero. Reserve the space for them here. They will + # be filled next if needed. + warning_summary = "
" + failure_summary = "
" + manual_summary = "
" + error_summary = "
" + + if n_warn > 0: + warning_summary = (f"
{n_warn}" + f" {pluralize('warning', 'warnings', n_warn)}
") + if n_fail > 0: + failure_summary = (f"
{n_fail}" + f" {pluralize('test', 'tests', n_fail)} failed
") + if n_manual > 0: + manual_summary = (f"
{n_manual} manual" + f" {pluralize('check', 'checks', n_manual)} needed
") + if n_error > 0: + error_summary = (f"
{n_error}" + f" {pluralize('error', 'errors', n_error)}
") + + return f"{pass_summary}{warning_summary}{failure_summary}{manual_summary}{error_summary}" + def run_reporter(args): """ Creates the indvididual reports and the front page @@ -203,41 +239,9 @@ def run_reporter(args): full_name = prod_to_fullname[product] link_path = "./IndividualReports/" f"{product_capitalize}Report.html" link = f"{full_name}" - ## Build the "Details" column - n_success = stats["Pass"] - n_warn = stats["Warning"] - n_fail = stats["Fail"] - n_manual = stats["N/A"] + stats["No events found"] - n_error = stats["Error"] - - pass_summary = (f"
{n_success}" - f" {pluralize('test', 'tests', n_success)} passed
") - - # The warnings, failures, and manuals are only shown if they are - # greater than zero. Reserve the space for them here. They will - # be filled next if needed. - warning_summary = "
" - failure_summary = "
" - manual_summary = "
" - error_summary = "
" - - if n_warn > 0: - warning_summary = (f"
{n_warn}" - f" {pluralize('warning', 'warnings', n_warn)}
") - if n_fail > 0: - failure_summary = (f"
{n_fail}" - f" {pluralize('test', 'tests', n_fail)} failed
") - if n_manual > 0: - manual_summary = (f"
{n_manual} manual" - f" {pluralize('check', 'checks', n_manual)} needed
") - if n_error > 0: - error_summary = (f"
{n_error}" - f" {pluralize('error', 'errors', n_error)}
") - table_data.append({ "Baseline Conformance Reports": link, - "Details": f"{pass_summary}{warning_summary}{failure_summary}{manual_summary}\ - {error_summary}" + "Details": generate_summary(stats) }) fragments.append(reporter.create_html_table(table_data))