From 59468fd5db925f68e07e00e5b5354e83e0c6f0b1 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 23 Jan 2024 15:58:14 -0800
Subject: [PATCH 01/35] Implement warning for missing output

---
 scubagoggles/orchestrator.py          |  10 ++-
 scubagoggles/reporter/reporter.py     | 103 +++++++++++++++-----------
 scubagoggles/reporter/scripts/main.js |   4 +-
 3 files changed, 71 insertions(+), 46 deletions(-)

diff --git a/scubagoggles/orchestrator.py b/scubagoggles/orchestrator.py
index be3d4188..e65860ae 100644
--- a/scubagoggles/orchestrator.py
+++ b/scubagoggles/orchestrator.py
@@ -208,6 +208,7 @@ def run_reporter(args):
         n_warn = stats["Warning"]
         n_fail = stats["Fail"]
         n_manual = stats["N/A"] + stats["No events found"]
+        n_error = stats["Error"]
 
         pass_summary = (f"<div class='summary pass'>{n_success}"
         f" {pluralize('test', 'tests', n_success)} passed</div>")
@@ -218,6 +219,7 @@ def run_reporter(args):
         warning_summary = "<div class='summary'></div>"
         failure_summary = "<div class='summary'></div>"
         manual_summary = "<div class='summary'></div>"
+        error_summary = "<div class='summary'></div>"
 
         if n_warn > 0:
             warning_summary = (f"<div class='summary warning'>{n_warn}"
@@ -228,10 +230,14 @@ def run_reporter(args):
         if n_manual > 0:
             manual_summary = (f"<div class='summary manual'>{n_manual} manual"
             f" {pluralize('check', 'checks', n_manual)} needed</div>")
+        if n_error > 0:
+            error_summary = (f"<div class='summary error'>{n_error}"
+            f" {pluralize('error', 'errors', n_error)}</div>")
 
         table_data.append({
-        "Baseline Conformance Reports": link,
-        "Details": f"{pass_summary}{warning_summary}{failure_summary}{manual_summary}"
+            "Baseline Conformance Reports": link,
+            "Details": f"{pass_summary}{warning_summary}{failure_summary}{manual_summary}\
+                {error_summary}"
         })
 
     fragments.append(reporter.create_html_table(table_data))
diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index 57065b7f..564c0f75 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -5,10 +5,13 @@
 """
 import os
 import time
+import warnings
 from datetime import datetime
 import pandas as pd
 from scubagoggles.utils import rel_abs_path
 
+SCUBA_GITHUB_URL = "https://github.com/cisagov/scubagoggles"
+
 def get_test_result(requirement_met : bool, criticality : str, no_such_events : bool) -> str:
     '''
     Checks the Rego to see if the baseline passed or failed and indicates the criticality
@@ -164,56 +167,72 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str,
         "Warning": 0,
         "Fail": 0,
         "N/A": 0,
-        "No events found": 0
+        "No events found": 0,
+        "Error": 0
     }
 
     for baseline_group in product_policies:
         table_data = []
         for control in baseline_group['Controls']:
             tests = [test for test in test_results_data if test['PolicyId'] == control['Id']]
-            for test in tests:
-                result = get_test_result(test['RequirementMet'], test['Criticality'],
-                test['NoSuchEvent'])
-                report_stats[result] = report_stats[result] + 1
-                details = test['ReportDetails']
-
-                if result == "No events found":
-                    warning_icon = "<object data='./images/triangle-exclamation-solid.svg'\
-                        width='15'\
-                        height='15'>\
-                        </object>"
-                    details = warning_icon + " " + test['ReportDetails']
-
-                # As rules doesn't have it's own baseline, Rules and Common Controls
-                # need to be handled specially
-                if product_capitalized == "Rules":
-                    if 'Not-Implemented' in test['Criticality']:
-                        # The easiest way to identify the GWS.COMMONCONTROLS.14.1v1
-                        # results that belong to the Common Controls report is they're
-                        # marked as Not-Implemented. This if excludes them from the
-                        # rules report.
-                        continue
-                    table_data.append({
-                        'Control ID': control['Id'],
-                        'Rule Name': test['Requirement'],
-                        'Result': result,
-                        'Criticality': test['Criticality'],
-                        'Rule Description': test['ReportDetails']})
-                elif product_capitalized == "Commoncontrols" \
-                    and baseline_group['GroupName'] == 'System-defined Rules' \
-                    and 'Not-Implemented' not in test['Criticality']:
-                    # The easiest way to identify the System-defined Rules
-                    # results that belong to the Common Controls report is they're
-                    # marked as Not-Implemented. This if excludes the full results
-                    # from the Common Controls report.
-                    continue
-                else:
-                    table_data.append({
+            if len(tests) == 0:
+                # Handle the case where Rego doesn't output anything for a given control
+                report_stats['Error'] += 1
+                issues_link = f'<a href="{SCUBA_GITHUB_URL}/issues" target="_blank">GitHub</a>'
+                table_data.append({
                     'Control ID': control['Id'],
                     'Requirement': control['Value'],
-                    'Result': result,
-                    'Criticality': test['Criticality'],
-                    'Details': details})
+                    'Result': "Error - Test results missing",
+                    'Criticality': "-",
+                    'Details': f'Report issue on {issues_link}'
+                })
+                warnings.warn(f"No test results found for Control Id {control['Id']}",
+                    RuntimeWarning)
+            else:
+                for test in tests:
+                    result = get_test_result(test['RequirementMet'], test['Criticality'],
+                        test['NoSuchEvent'])
+                    report_stats[result] = report_stats[result] + 1
+                    details = test['ReportDetails']
+
+                    if result == "No events found":
+                        warning_icon = "<object data='./images/triangle-exclamation-solid.svg'\
+                            width='15'\
+                            height='15'>\
+                            </object>"
+                        details = warning_icon + " " + test['ReportDetails']
+
+                    # As rules doesn't have its own baseline, Rules and Common Controls
+                    # need to be handled specially
+                    if product_capitalized == "Rules":
+                        if 'Not-Implemented' in test['Criticality']:
+                            # The easiest way to identify the GWS.COMMONCONTROLS.13.1v1
+                            # results that belong to the Common Controls report is they're
+                            # marked as Not-Implemented. This if excludes them from the
+                            # rules report.
+                            continue
+                        table_data.append({
+                            'Control ID': control['Id'],
+                            'Rule Name': test['Requirement'],
+                            'Result': result,
+                            'Criticality': test['Criticality'],
+                            'Rule Description': test['ReportDetails']})
+                    elif product_capitalized == "Commoncontrols" \
+                        and baseline_group['GroupName'] == 'System-defined Rules' \
+                        and 'Not-Implemented' not in test['Criticality']:
+                        # The easiest way to identify the System-defined Rules
+                        # results that belong to the Common Controls report is they're
+                        # marked as Not-Implemented. This if excludes the full results
+                        # from the Common Controls report.
+                        continue
+                    else:
+                        table_data.append({
+                            'Control ID': control['Id'],
+                            'Requirement': control['Value'],
+                            'Result': result,
+                            'Criticality': test['Criticality'],
+                            'Details': details
+                        })
         fragments.append(f"<h2>{product_upper}-{baseline_group['GroupNumber']} \
         {baseline_group['GroupName']}</h2>")
         fragments.append(create_html_table(table_data))
diff --git a/scubagoggles/reporter/scripts/main.js b/scubagoggles/reporter/scripts/main.js
index d03f19a6..b48881f8 100644
--- a/scubagoggles/reporter/scripts/main.js
+++ b/scubagoggles/reporter/scripts/main.js
@@ -32,8 +32,8 @@ const colorRows = () => {
             }
             else if (rows[i].children[statusCol].innerHTML.includes("Error")) {
                 rows[i].style.background = "var(--test-fail)";
-                rows[i].querySelectorAll('td')[1].style.borderColor = "var(--border-color)";
-                rows[i].querySelectorAll('td')[1].style.color = "#d10000";
+                rows[i].querySelectorAll('td')[statusCol].style.borderColor = "var(--border-color)";
+                rows[i].querySelectorAll('td')[statusCol].style.color = "#d10000";
             }
         }
         catch (error) {

From d2c5fe6cd9c282b4fbce7a1769eed628f64ad0cd Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 23 Jan 2024 16:09:18 -0800
Subject: [PATCH 02/35] Add missing reo spf check

---
 Testing/RegoTests/gmail/gmail03_test.rego | 30 ++++++++++++++++++++---
 rego/Gmail.rego                           | 15 +++++++++++-
 2 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/Testing/RegoTests/gmail/gmail03_test.rego b/Testing/RegoTests/gmail/gmail03_test.rego
index 199c8ac6..4cd04437 100644
--- a/Testing/RegoTests/gmail/gmail03_test.rego
+++ b/Testing/RegoTests/gmail/gmail03_test.rego
@@ -5,9 +5,31 @@ import future.keywords
 #
 # GWS.GMAIL.3.1v0.1
 #--
+test_MaintainList_Correct_V1 if {
+    # Test not implemented
+    PolicyId := "GWS.GMAIL.17.1v0.1"
+    Output := tests with input as {
+        "gmail_logs": {"items": [
+        ]},
+        "tenant_info": {
+            "topLevelOU": ""
+        }
+    }
+
+    RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId]
+    count(RuleOutput) == 1
+    not RuleOutput[0].RequirementMet
+    not RuleOutput[0].NoSuchEvent
+    RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
+}
+#--
+
+#
+# GWS.GMAIL.3.2v0.1
+#--
 test_SPF_Correct_V1 if {
     # Test SPF when there's only one domain
-    PolicyId := "GWS.GMAIL.3.1v0.1"
+    PolicyId := "GWS.GMAIL.3.2v0.1"
     Output := tests with input as {
         "dkim_records": [
             {
@@ -32,7 +54,7 @@ test_SPF_Correct_V1 if {
 
 test_SPF_Correct_V2 if {
     # Test SPF when there's multiple domains
-    PolicyId := "GWS.GMAIL.3.1v0.1"
+    PolicyId := "GWS.GMAIL.3.2v0.1"
     Output := tests with input as {
         "dkim_records": [
             {
@@ -65,7 +87,7 @@ test_SPF_Correct_V2 if {
 
 test_SPF_Incorrect_V1 if {
     # Test SPF when there's multiple domains and only one is correct
-    PolicyId := "GWS.GMAIL.3.1v0.1"
+    PolicyId := "GWS.GMAIL.3.2v0.1"
     Output := tests with input as {
         "dkim_records": [
             {
@@ -98,7 +120,7 @@ test_SPF_Incorrect_V1 if {
 
 test_SPF_Incorrect_V2 if {
     # Test SPF when there's only one domain and it's wrong
-    PolicyId := "GWS.GMAIL.3.1v0.1"
+    PolicyId := "GWS.GMAIL.3.2v0.1"
     Output := tests with input as {
         "dkim_records": [
             {
diff --git a/rego/Gmail.rego b/rego/Gmail.rego
index b7a93419..238d6249 100644
--- a/rego/Gmail.rego
+++ b/rego/Gmail.rego
@@ -107,6 +107,19 @@ if {
 #
 # Baseline GWS.GMAIL.3.1v0.1
 #--
+# No implementation steps provided for this policy
+tests contains {
+    "PolicyId": "GWS.GMAIL.3.1v0.1",
+    "Criticality": "Shall/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false}
+#--
+
+#
+# Baseline GWS.GMAIL.3.2v0.1
+#--
 DomainsWithSpf contains SpfRecord.domain if {
     some SpfRecord in input.spf_records
     some Rdata in SpfRecord.rdata
@@ -114,7 +127,7 @@ DomainsWithSpf contains SpfRecord.domain if {
 }
 
 tests contains {
-    "PolicyId": "GWS.GMAIL.3.1v0.1",
+    "PolicyId": "GWS.GMAIL.3.2v0.1",
     "Criticality": "Shall",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutSpf, AllDomains),
     "ActualValue": DomainsWithoutSpf,

From 74ce7675ccdde9ae58811bb8b362f12372dd1c84 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 23 Jan 2024 16:13:48 -0800
Subject: [PATCH 03/35] Add rego check for 10.2

---
 Testing/RegoTests/gmail/gmail10_test.rego | 20 ++++++++++++++++++++
 rego/Gmail.rego                           | 12 ++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/Testing/RegoTests/gmail/gmail10_test.rego b/Testing/RegoTests/gmail/gmail10_test.rego
index 00a3f2b7..3609a40b 100644
--- a/Testing/RegoTests/gmail/gmail10_test.rego
+++ b/Testing/RegoTests/gmail/gmail10_test.rego
@@ -310,5 +310,25 @@ test_GoogleWorkspaceSync_Incorrect_V5 if {
     not RuleOutput[0].NoSuchEvent
     RuleOutput[0].ReportDetails == "Requirement failed in Secondary OU."
 }
+#--
 
+#
+# GWS.GMAIL.10.2v0.1
+test_May_Correct_V1 if {
+    # Test Comprehensive Mail Storage when there's only one event
+    PolicyId := "GWS.GMAIL.10.2v0.1"
+    Output := tests with input as {
+        "gmail_logs": {"items": [
+        ]},
+        "tenant_info": {
+            "topLevelOU": ""
+        }
+    }
+
+    RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId]
+    count(RuleOutput) == 1
+    not RuleOutput[0].RequirementMet
+    not RuleOutput[0].NoSuchEvent
+    RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
+}
 #--
\ No newline at end of file
diff --git a/rego/Gmail.rego b/rego/Gmail.rego
index 238d6249..681f289f 100644
--- a/rego/Gmail.rego
+++ b/rego/Gmail.rego
@@ -1362,6 +1362,18 @@ if {
 }
 #--
 
+#
+# Baseline GWS.GMAIL.10.2v0.1
+#--
+# No implementation steps provided for this policy
+tests contains {
+    "PolicyId": "GWS.GMAIL.10.2v0.1",
+    "Criticality": "May/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false}
+#--
 
 ################
 # GWS.GMAIL.11 #

From 33ad8979e3d855f2e6a856cd9dc6a8a66ebdd86d Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 23 Jan 2024 16:17:48 -0800
Subject: [PATCH 04/35] Add rego check for 15.2

---
 Testing/RegoTests/gmail/gmail10_test.rego |  3 ++-
 Testing/RegoTests/gmail/gmail15_test.rego | 22 ++++++++++++++++++++++
 rego/Gmail.rego                           | 22 +++++++++++++++++++---
 3 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/Testing/RegoTests/gmail/gmail10_test.rego b/Testing/RegoTests/gmail/gmail10_test.rego
index 3609a40b..8e0aa7d6 100644
--- a/Testing/RegoTests/gmail/gmail10_test.rego
+++ b/Testing/RegoTests/gmail/gmail10_test.rego
@@ -314,8 +314,9 @@ test_GoogleWorkspaceSync_Incorrect_V5 if {
 
 #
 # GWS.GMAIL.10.2v0.1
+#--
 test_May_Correct_V1 if {
-    # Test Comprehensive Mail Storage when there's only one event
+    # Test not implemented
     PolicyId := "GWS.GMAIL.10.2v0.1"
     Output := tests with input as {
         "gmail_logs": {"items": [
diff --git a/Testing/RegoTests/gmail/gmail15_test.rego b/Testing/RegoTests/gmail/gmail15_test.rego
index 14ca6145..eadd92dc 100644
--- a/Testing/RegoTests/gmail/gmail15_test.rego
+++ b/Testing/RegoTests/gmail/gmail15_test.rego
@@ -348,4 +348,26 @@ test_EnhancedPreDeliveryMessageScanning_Incorrect_V5 if {
     not RuleOutput[0].NoSuchEvent
     RuleOutput[0].ReportDetails == "Requirement failed in Secondary OU."
 }
+#--
+
+#
+# GWS.GMAIL.15.2v0.1
+#--
+test_Other_Correct_V1 if {
+    # Test not implemented
+    PolicyId := "GWS.GMAIL.15.2v0.1"
+    Output := tests with input as {
+        "gmail_logs": {"items": [
+        ]},
+        "tenant_info": {
+            "topLevelOU": ""
+        }
+    }
+
+    RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId]
+    count(RuleOutput) == 1
+    not RuleOutput[0].RequirementMet
+    not RuleOutput[0].NoSuchEvent
+    RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
+}
 #--
\ No newline at end of file
diff --git a/rego/Gmail.rego b/rego/Gmail.rego
index 681f289f..5c845e1c 100644
--- a/rego/Gmail.rego
+++ b/rego/Gmail.rego
@@ -114,7 +114,8 @@ tests contains {
     "ReportDetails": "Currently not able to be tested automatically; please manually check.",
     "ActualValue": "",
     "RequirementMet": false,
-    "NoSuchEvent": false}
+    "NoSuchEvent": false
+}
 #--
 
 #
@@ -530,7 +531,8 @@ tests contains {
     "ReportDetails": "Currently not able to be tested automatically; please manually check.",
     "ActualValue": "",
     "RequirementMet": false,
-    "NoSuchEvent": false}
+    "NoSuchEvent": false
+}
 #--
 
 ###############
@@ -1372,7 +1374,8 @@ tests contains {
     "ReportDetails": "Currently not able to be tested automatically; please manually check.",
     "ActualValue": "",
     "RequirementMet": false,
-    "NoSuchEvent": false}
+    "NoSuchEvent": false
+}
 #--
 
 ################
@@ -1633,6 +1636,19 @@ if {
 }
 #--
 
+#
+# Baseline GWS.GMAIL.15.2v0.1
+#--
+# No implementation steps provided for this policy
+tests contains {
+    "PolicyId": "GWS.GMAIL.15.2v0.1",
+    "Criticality": "Should/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false
+}
+#--
 
 ################
 # GWS.GMAIL.16 #

From 09d6e05b72293ea9b40d4d200745a2b67c400567 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 23 Jan 2024 16:19:28 -0800
Subject: [PATCH 05/35] Add rego check for 16.2

---
 Testing/RegoTests/gmail/gmail16_test.rego | 22 ++++++++++++++++++++++
 rego/Gmail.rego                           | 13 +++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/Testing/RegoTests/gmail/gmail16_test.rego b/Testing/RegoTests/gmail/gmail16_test.rego
index fdda42ab..e0bf7a02 100644
--- a/Testing/RegoTests/gmail/gmail16_test.rego
+++ b/Testing/RegoTests/gmail/gmail16_test.rego
@@ -310,4 +310,26 @@ test_SecuritySandbox_Incorrect_V5 if {
     not RuleOutput[0].NoSuchEvent
     RuleOutput[0].ReportDetails == "Requirement failed in Secondary OU."
 }
+#--
+
+#
+# GWS.GMAIL.16.2v0.1
+#--
+test_Other_Correct_V1 if {
+    # Test not implemented
+    PolicyId := "GWS.GMAIL.16.2v0.1"
+    Output := tests with input as {
+        "gmail_logs": {"items": [
+        ]},
+        "tenant_info": {
+            "topLevelOU": ""
+        }
+    }
+
+    RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId]
+    count(RuleOutput) == 1
+    not RuleOutput[0].RequirementMet
+    not RuleOutput[0].NoSuchEvent
+    RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
+}
 #--
\ No newline at end of file
diff --git a/rego/Gmail.rego b/rego/Gmail.rego
index 5c845e1c..913b18a3 100644
--- a/rego/Gmail.rego
+++ b/rego/Gmail.rego
@@ -1701,6 +1701,19 @@ if {
 }
 #--
 
+#
+# Baseline GWS.GMAIL.16.2v0.1
+#--
+# No implementation steps provided for this policy
+tests contains {
+    "PolicyId": "GWS.GMAIL.16.2v0.1",
+    "Criticality": "Should/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false
+}
+#--
 
 ################
 # GWS.GMAIL.17 #

From 9156aadab6f69b5a59594d76903683c365c0f0e9 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 23 Jan 2024 16:23:44 -0800
Subject: [PATCH 06/35] Add missing rego checks for group 18

---
 Testing/RegoTests/gmail/gmail18_test.rego | 44 +++++++++++++++++++++++
 rego/Gmail.rego                           | 27 ++++++++++++++
 2 files changed, 71 insertions(+)

diff --git a/Testing/RegoTests/gmail/gmail18_test.rego b/Testing/RegoTests/gmail/gmail18_test.rego
index 72ecced6..b1f63d3d 100644
--- a/Testing/RegoTests/gmail/gmail18_test.rego
+++ b/Testing/RegoTests/gmail/gmail18_test.rego
@@ -22,4 +22,48 @@ test_AdvanvedEmailContentFitlering_Correct_V1 if {
     not RuleOutput[0].NoSuchEvent
     RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
 }
+#--
+
+#
+# GWS.GMAIL.18.2v0.1
+#--
+test_Other_Correct_V1 if {
+    # Test not implemented
+    PolicyId := "GWS.GMAIL.18.2v0.1"
+    Output := tests with input as {
+        "gmail_logs": {"items": [
+        ]},
+        "tenant_info": {
+            "topLevelOU": ""
+        }
+    }
+
+    RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId]
+    count(RuleOutput) == 1
+    not RuleOutput[0].RequirementMet
+    not RuleOutput[0].NoSuchEvent
+    RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
+}
+#--
+
+#
+# GWS.GMAIL.18.3v0.1
+#--
+test_PII_Correct_V1 if {
+    # Test not implemented
+    PolicyId := "GWS.GMAIL.18.3v0.1"
+    Output := tests with input as {
+        "gmail_logs": {"items": [
+        ]},
+        "tenant_info": {
+            "topLevelOU": ""
+        }
+    }
+
+    RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId]
+    count(RuleOutput) == 1
+    not RuleOutput[0].RequirementMet
+    not RuleOutput[0].NoSuchEvent
+    RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
+}
 #--
\ No newline at end of file
diff --git a/rego/Gmail.rego b/rego/Gmail.rego
index 913b18a3..63fc6f61 100644
--- a/rego/Gmail.rego
+++ b/rego/Gmail.rego
@@ -1754,6 +1754,33 @@ tests contains {
 }
 #--
 
+#
+# Baseline GWS.GMAIL.18.2v0.1
+#--
+# No implementation steps provided for this policy
+tests contains {
+    "PolicyId": "GWS.GMAIL.18.2v0.1",
+    "Criticality": "Should/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false
+}
+#--
+
+#
+# Baseline GWS.GMAIL.18.3v0.1
+#--
+# No implementation steps provided for this policy
+tests contains {
+    "PolicyId": "GWS.GMAIL.18.3v0.1",
+    "Criticality": "Shall/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false
+}
+#--
 
 ################
 # GWS.GMAIL.19 #

From e358c87713d15413429778854766163c0af6a84b Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 23 Jan 2024 16:40:46 -0800
Subject: [PATCH 07/35] Broke up the run_reporter function to satisfy linter

---
 scubagoggles/orchestrator.py | 70 +++++++++++++++++++-----------------
 1 file changed, 37 insertions(+), 33 deletions(-)

diff --git a/scubagoggles/orchestrator.py b/scubagoggles/orchestrator.py
index e65860ae..b3815b60 100644
--- a/scubagoggles/orchestrator.py
+++ b/scubagoggles/orchestrator.py
@@ -119,6 +119,42 @@ def pluralize(singular : str, plural : str, count : int) -> str:
         return singular
     return plural
 
+def generate_summary(stats : dict) -> str:
+    """
+    Craft the html-formatted summary from the stats dictionary.
+    """
+    n_success = stats["Pass"]
+    n_warn = stats["Warning"]
+    n_fail = stats["Fail"]
+    n_manual = stats["N/A"] + stats["No events found"]
+    n_error = stats["Error"]
+
+    pass_summary = (f"<div class='summary pass'>{n_success}"
+    f" {pluralize('test', 'tests', n_success)} passed</div>")
+
+    # The warnings, failures, and manuals are only shown if they are
+    # greater than zero. Reserve the space for them here. They will
+    # be filled next if needed.
+    warning_summary = "<div class='summary'></div>"
+    failure_summary = "<div class='summary'></div>"
+    manual_summary = "<div class='summary'></div>"
+    error_summary = "<div class='summary'></div>"
+
+    if n_warn > 0:
+        warning_summary = (f"<div class='summary warning'>{n_warn}"
+        f" {pluralize('warning', 'warnings', n_warn)}</div>")
+    if n_fail > 0:
+        failure_summary = (f"<div class='summary failure'>{n_fail}"
+        f" {pluralize('test', 'tests', n_fail)} failed</div>")
+    if n_manual > 0:
+        manual_summary = (f"<div class='summary manual'>{n_manual} manual"
+        f" {pluralize('check', 'checks', n_manual)} needed</div>")
+    if n_error > 0:
+        error_summary = (f"<div class='summary error'>{n_error}"
+        f" {pluralize('error', 'errors', n_error)}</div>")
+
+    return f"{pass_summary}{warning_summary}{failure_summary}{manual_summary}{error_summary}"
+
 def run_reporter(args):
     """
     Creates the indvididual reports and the front page
@@ -203,41 +239,9 @@ def run_reporter(args):
         full_name = prod_to_fullname[product]
         link_path =  "./IndividualReports/" f"{product_capitalize}Report.html"
         link = f"<a class=\"individual_reports\" href={link_path}>{full_name}</a>"
-        ## Build the "Details" column
-        n_success = stats["Pass"]
-        n_warn = stats["Warning"]
-        n_fail = stats["Fail"]
-        n_manual = stats["N/A"] + stats["No events found"]
-        n_error = stats["Error"]
-
-        pass_summary = (f"<div class='summary pass'>{n_success}"
-        f" {pluralize('test', 'tests', n_success)} passed</div>")
-
-        # The warnings, failures, and manuals are only shown if they are
-        # greater than zero. Reserve the space for them here. They will
-        # be filled next if needed.
-        warning_summary = "<div class='summary'></div>"
-        failure_summary = "<div class='summary'></div>"
-        manual_summary = "<div class='summary'></div>"
-        error_summary = "<div class='summary'></div>"
-
-        if n_warn > 0:
-            warning_summary = (f"<div class='summary warning'>{n_warn}"
-            f" {pluralize('warning', 'warnings', n_warn)}</div>")
-        if n_fail > 0:
-            failure_summary = (f"<div class='summary failure'>{n_fail}"
-            f" {pluralize('test', 'tests', n_fail)} failed</div>")
-        if n_manual > 0:
-            manual_summary = (f"<div class='summary manual'>{n_manual} manual"
-            f" {pluralize('check', 'checks', n_manual)} needed</div>")
-        if n_error > 0:
-            error_summary = (f"<div class='summary error'>{n_error}"
-            f" {pluralize('error', 'errors', n_error)}</div>")
-
         table_data.append({
             "Baseline Conformance Reports": link,
-            "Details": f"{pass_summary}{warning_summary}{failure_summary}{manual_summary}\
-                {error_summary}"
+            "Details": generate_summary(stats)
         })
 
     fragments.append(reporter.create_html_table(table_data))

From c2770e6c5b19decfe58f4118157ec3bdabbed6a3 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Wed, 24 Jan 2024 17:50:59 -0800
Subject: [PATCH 08/35] Initial commit of error handling code

---
 rego/Commoncontrols.rego          |   1 +
 scubagoggles/orchestrator.py      |  20 +-
 scubagoggles/provider.py          | 850 ++++++++++++++++--------------
 scubagoggles/reporter/reporter.py | 139 +++--
 4 files changed, 557 insertions(+), 453 deletions(-)

diff --git a/rego/Commoncontrols.rego b/rego/Commoncontrols.rego
index db577ac7..df2d1278 100644
--- a/rego/Commoncontrols.rego
+++ b/rego/Commoncontrols.rego
@@ -713,6 +713,7 @@ tests contains {
 #--
 tests contains {
     "PolicyId": "GWS.COMMONCONTROLS.7.1v0.1",
+    "Prerequisites": ["directory/v1/users/list"],
     "Criticality": "Shall",
     "ReportDetails": concat("", [
         concat("", ["The following super admins are configured: ", concat(", ", SuperAdmins)]),
diff --git a/scubagoggles/orchestrator.py b/scubagoggles/orchestrator.py
index be3d4188..1fb31a31 100644
--- a/scubagoggles/orchestrator.py
+++ b/scubagoggles/orchestrator.py
@@ -12,7 +12,7 @@
 from googleapiclient.discovery import build
 
 from scubagoggles.auth import gws_auth
-from scubagoggles.provider import call_gws_providers
+from scubagoggles.provider import Provider
 from scubagoggles.run_rego import opa_eval
 from scubagoggles.reporter import reporter, md_parser
 from scubagoggles.utils import rel_abs_path
@@ -63,8 +63,10 @@ def run_gws_providers(args, services):
     out_folder = args.outputpath
     provider_dict = {}
 
-    provider_dict = call_gws_providers(products, services, args.quiet)
-
+    provider = Provider()
+    provider_dict = provider.call_gws_providers(products, services, args.quiet)
+    provider_dict['successful_calls'] = list(provider.successful_calls)
+    provider_dict['unsuccessful_calls'] = list(provider.unsuccessful_calls)
     settings_json = json.dumps(provider_dict, indent = 4)
     out_path = out_folder + f'/{args.outputproviderfilename}.json'
     with open(out_path, mode="w", encoding='UTF-8') as outfile:
@@ -146,8 +148,14 @@ def run_reporter(args):
     with open(test_results_json, mode='r', encoding='UTF-8') as file:
         test_results_data = json.load(file)
 
-    # baseline_path
+    # Get the successful/unsuccessful commands
+    settings_name = f'{out_folder}/{args.outputproviderfilename}.json'
+    with open(settings_name, mode='r', encoding='UTF-8') as file:
+        settings_data = json.load(file)
+    successful_calls = set(settings_data['successful_calls'])
+    unsuccessful_calls = set(settings_data['unsuccessful_calls'])
 
+    # baseline_path
     subset_prod_to_fullname = {
         key: prod_to_fullname[key]
         for key in args.baselines
@@ -188,7 +196,9 @@ def run_reporter(args):
             tenant_name,
             main_report_name,
             prod_to_fullname,
-            baseline_policies[product]
+            baseline_policies[product],
+            successful_calls,
+            unsuccessful_calls
         )
 
     # Make the report front page
diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index adb07529..aa9c0ff7 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -86,424 +86,456 @@
 
 DNSClient = RobustDNSClient()
 
-def get_spf_records(domains: list) -> list:
-    '''
-    Gets the SPF records for each domain in domains.
-
-    :param domains: The list of domain names (strings).
-    '''
-    results = []
-    n_low_confidence = 0
-    for domain in domains:
-        result = DNSClient.query(domain)
-        if not result['HighConfidence']:
-            n_low_confidence += 1
-        results.append({
-            "domain": domain,
-            "rdata": result["Answers"],
-            "log": result["LogEntries"]
-        })
-    if n_low_confidence > 0:
-        warnings.warn(f"get_spf_records: for {n_low_confidence} domain(s), \
-the traditional DNS queries returned an empty answer \
-section and the DoH queries failed. Will assume SPF not configured, but \
-can't guarantee that failure isn't due to something like split horizon DNS. \
-See ProviderSettingsExport.json under 'spf_records' for more details.", RuntimeWarning)
-    return results
-
-def get_dkim_records(domains : list) -> list:
-    '''
-    Gets the DKIM records for each domain in domains.
-
-    :param domains: The list of domain names (strings).
-    '''
-    results = []
-    n_low_confidence = 0
-    for domain in domains:
-        qnames = [f"{selector}._domainkey.{domain}" for selector in selectors]
-        log_entries = []
-        for qname in qnames:
+class Provider:
+    def __init__(self):
+        self.successful_calls = set()
+        self.unsuccessful_calls = set()
+
+    def get_spf_records(self, domains: list) -> list:
+        '''
+        Gets the SPF records for each domain in domains.
+
+        :param domains: The list of domain names (strings).
+        '''
+        results = []
+        n_low_confidence = 0
+        for domain in domains:
+            result = DNSClient.query(domain)
+            if not result['HighConfidence']:
+                n_low_confidence += 1
+            results.append({
+                "domain": domain,
+                "rdata": result["Answers"],
+                "log": result["LogEntries"]
+            })
+        if n_low_confidence > 0:
+            warnings.warn(f"get_spf_records: for {n_low_confidence} domain(s), \
+    the traditional DNS queries returned an empty answer \
+    section and the DoH queries failed. Will assume SPF not configured, but \
+    can't guarantee that failure isn't due to something like split horizon DNS. \
+    See ProviderSettingsExport.json under 'spf_records' for more details.", RuntimeWarning)
+        return results
+
+    def get_dkim_records(self, domains : list) -> list:
+        '''
+        Gets the DKIM records for each domain in domains.
+
+        :param domains: The list of domain names (strings).
+        '''
+        results = []
+        n_low_confidence = 0
+        for domain in domains:
+            qnames = [f"{selector}._domainkey.{domain}" for selector in selectors]
+            log_entries = []
+            for qname in qnames:
+                result = DNSClient.query(qname)
+                log_entries.extend(result['LogEntries'])
+                if len(result['Answers']) == 0:
+                    # The DKIM record does not exist with this selector, we need to try again with
+                    # a different one
+                    continue
+                # Otherwise, the DKIM record exists with this selector, no need to try the rest
+                break
+
+            if not result['HighConfidence']:
+                n_low_confidence += 1
+            results.append({
+                "domain": domain,
+                "rdata": result["Answers"],
+                "log": log_entries
+            })
+
+        if n_low_confidence > 0:
+            warnings.warn(f"get_dkim_records: for {n_low_confidence} domain(s), \
+    the traditional DNS queries returned an empty answer \
+    section and the DoH queries failed. Will assume DKIM not configured, but \
+    can't guarantee that failure isn't due to something like split horizon DNS. \
+    See ProviderSettingsExport.json under 'dkim_records' for more details.", RuntimeWarning)
+        return results
+
+    def get_dmarc_records(self, domains : list) -> list:
+        '''
+        Gets the DMARC records for each domain in domains.
+
+        :param domains: The list of domain names (strings).
+        '''
+        results = []
+        n_low_confidence = 0
+        for domain in domains:
+            log_entries = []
+            qname = f"_dmarc.{domain}"
             result = DNSClient.query(qname)
             log_entries.extend(result['LogEntries'])
-            if len(result['Answers']) == 0:
-                # The DKIM record does not exist with this selector, we need to try again with
-                # a different one
-                continue
-            # Otherwise, the DKIM record exists with this selector, no need to try the rest
-            break
-
-        if not result['HighConfidence']:
-            n_low_confidence += 1
-        results.append({
-            "domain": domain,
-            "rdata": result["Answers"],
-            "log": log_entries
-        })
-
-    if n_low_confidence > 0:
-        warnings.warn(f"get_dkim_records: for {n_low_confidence} domain(s), \
-the traditional DNS queries returned an empty answer \
-section and the DoH queries failed. Will assume DKIM not configured, but \
-can't guarantee that failure isn't due to something like split horizon DNS. \
-See ProviderSettingsExport.json under 'dkim_records' for more details.", RuntimeWarning)
-    return results
-
-def get_dmarc_records(domains : list) -> list:
-    '''
-    Gets the DMARC records for each domain in domains.
-
-    :param domains: The list of domain names (strings).
-    '''
-    results = []
-    n_low_confidence = 0
-    for domain in domains:
-        log_entries = []
-        qname = f"_dmarc.{domain}"
-        result = DNSClient.query(qname)
-        log_entries.extend(result['LogEntries'])
-        if len(result["Answers"]) == 0:
-            # The domain does not exist. If the record is not available at the full domain
-            # level, we need to check at the organizational domain level.
-            labels = domain.split(".")
-            org_domain = f"{labels[-2]}.{labels[-1]}"
-            result = DNSClient.query(f"_dmarc.{org_domain}")
-            log_entries.extend(result['LogEntries'])
-        if not result['HighConfidence']:
-            n_low_confidence += 1
-        results.append({
-            "domain": domain,
-            "rdata": result["Answers"],
-            "log": log_entries
-        })
-    if n_low_confidence > 0:
-        warnings.warn(f"get_dmarc_records: for {n_low_confidence} domain(s), \
-the traditional DNS queries returned an empty answer \
-section and the DoH queries failed. Will assume DMARC not configured, but \
-can't guarantee that failure isn't due to something like split horizon DNS. \
-See ProviderSettingsExport.json under 'dmarc_records' for more details.", RuntimeWarning)
-    return results
-
-def get_dnsinfo(service):
-    '''
-    Gets DNS Information for Gmail baseline
-
-    :param service: a directory_v1 service instance
-    '''
-    output = {"domains": [], "spf_records": [], "dkim_records": [], "dmarc_records": []}
-
-    # Determine the tenant's domains via the API
-    response = service.domains().list(customer="my_customer").execute()
-    domains = {d['domainName'] for d in response['domains']}
-
-    if len(domains) == 0:
-        warnings.warn("No domains found.", RuntimeWarning)
+            if len(result["Answers"]) == 0:
+                # The domain does not exist. If the record is not available at the full domain
+                # level, we need to check at the organizational domain level.
+                labels = domain.split(".")
+                org_domain = f"{labels[-2]}.{labels[-1]}"
+                result = DNSClient.query(f"_dmarc.{org_domain}")
+                log_entries.extend(result['LogEntries'])
+            if not result['HighConfidence']:
+                n_low_confidence += 1
+            results.append({
+                "domain": domain,
+                "rdata": result["Answers"],
+                "log": log_entries
+            })
+        if n_low_confidence > 0:
+            warnings.warn(f"get_dmarc_records: for {n_low_confidence} domain(s), \
+    the traditional DNS queries returned an empty answer \
+    section and the DoH queries failed. Will assume DMARC not configured, but \
+    can't guarantee that failure isn't due to something like split horizon DNS. \
+    See ProviderSettingsExport.json under 'dmarc_records' for more details.", RuntimeWarning)
+        return results
+
+    def get_dnsinfo(self, service):
+        '''
+        Gets DNS Information for Gmail baseline
+
+        :param service: a directory_v1 service instance
+        '''
+        output = {"domains": [], "spf_records": [], "dkim_records": [], "dmarc_records": []}
+
+        # Determine the tenant's domains via the API
+        response = service.domains().list(customer="my_customer").execute()
+        domains = {d['domainName'] for d in response['domains']}
+
+        if len(domains) == 0:
+            warnings.warn("No domains found.", RuntimeWarning)
+            return output
+
+        output["domains"].extend(domains)
+
+        try:
+            output["spf_records"] = get_spf_records(domains)
+            self.successful_calls.add("get_spf_records")
+        except Exception as exc:
+            output["spf_records"] = []
+            warnings.warn(f"An exception was thrown by get_spf_records: {exc}", RuntimeWarning)
+            self.unsuccessful_calls.add("get_spf_records")
+        try:
+            output["dkim_records"] = get_dkim_records(domains)
+            self.successful_calls.add("get_dkim_records")
+        except Exception as exc:
+            output["dkim_records"] = []
+            warnings.warn(f"An exception was thrown by get_dkim_records: {exc}", RuntimeWarning)
+            self.unsuccessful_calls.add("get_dkim_records")
+        try:
+            output["dmarc_records"] = get_dmarc_records(domains)
+            self.successful_calls.add("get_dmarc_records")
+        except Exception as exc:
+            output["dmarc_records"] = []
+            warnings.warn(f"An exception was thrown by get_dmarc_records: {exc}", RuntimeWarning)
+            self.unsuccessful_calls.add("get_dmarc_records")
         return output
 
-    output["domains"].extend(domains)
-
-    try:
-        output["spf_records"] = get_spf_records(domains)
-    except Exception as exc:
-        output["spf_records"] = []
-        warnings.warn(f"An exception was thrown by get_spf_records: {exc}", RuntimeWarning)
-    try:
-        output["dkim_records"] = get_dkim_records(domains)
-    except Exception as exc:
-        output["dkim_records"] = []
-        warnings.warn(f"An exception was thrown by get_dkim_records: {exc}", RuntimeWarning)
-    try:
-        output["dmarc_records"] = get_dmarc_records(domains)
-    except Exception as exc:
-        output["dmarc_records"] = []
-        warnings.warn(f"An exception was thrown by get_dmarc_records: {exc}", RuntimeWarning)
-    return output
-
-def get_super_admins(service) -> dict:
-    '''
-    Gets the org unit/primary email of all super admins, using the directory API
-
-    :param service: a directory_v1 service instance
-    '''
-    try:
-        response = service.users().list(customer="my_customer", query="isAdmin=True").execute()
-        admins = []
-        for user in response['users']:
-            org_unit = user['orgUnitPath']
-            # strip out the leading '/'
-            org_unit = org_unit[1:] if org_unit.startswith('/') else org_unit
-            email = user['primaryEmail']
-            admins.append({'primaryEmail': email, 'orgUnitPath': org_unit})
-        return {'super_admins': admins}
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting super admins; outputs will be incorrect: {exc}",
-            RuntimeWarning
-        )
-        return {'super_admins': []}
-
-def get_ous(service) -> dict:
-    '''
-    Gets the organizational units using the directory API
-
-    :param service: a directory_v1 service instance
-    '''
-
-    try:
-        response = service.orgunits().list(customerId='my_customer').execute()
-        if 'organizationUnits' not in response:
+    def get_super_admins(self, service) -> dict:
+        '''
+        Gets the org unit/primary email of all super admins, using the directory API
+
+        :param service: a directory_v1 service instance
+        '''
+        try:
+            x/0
+            response = service.users().list(customer="my_customer", query="isAdmin=True").execute()
+            admins = []
+            for user in response['users']:
+                org_unit = user['orgUnitPath']
+                # strip out the leading '/'
+                org_unit = org_unit[1:] if org_unit.startswith('/') else org_unit
+                email = user['primaryEmail']
+                admins.append({'primaryEmail': email, 'orgUnitPath': org_unit})
+            self.successful_calls.add("directory/v1/users/list")
+            return {'super_admins': admins}
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting super admins; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/users/list")
+            return {'super_admins': []}
+
+    def get_ous(self, service) -> dict:
+        '''
+        Gets the organizational units using the directory API
+
+        :param service: a directory_v1 service instance
+        '''
+
+        try:
+            response = service.orgunits().list(customerId='my_customer').execute()
+            self.successful_calls.add("directory/v1/orgunits/list")
+            if 'organizationUnits' not in response:
+                return {}
+            return response
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting top level OU: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/orgunits/list")
             return {}
-        return response
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting top level OU: {exc}",
-            RuntimeWarning
-        )
-        return {}
-
-def get_toplevel_ou(service) -> str:
-    '''
-    Gets the tenant name using the directory API
-
-    :param service: a directory_v1 service instance
-    '''
-
-    try:
-        response = service.orgunits().list(customerId='my_customer',
-                                    orgUnitPath='/',
-                                    type='children').execute()
-        # Because we set orgUnitPath to / and type to children, the API call will only
-        # return the second-level OUs, meaning the parentOrgUnitId of any of the OUs returned
-        # will point us to OU of the entire organization
-        if 'organizationUnits' not in response:
-            # No custom OUs have been created. In this case, we can't
-            # determine the name of the top-level OU. See:
-            # https://stackoverflow.com/questions/26936357/google-directory-api-org-name-of-root-org-unit-path
-            # https://stackoverflow.com/questions/60464432/cannot-get-root-orgunit-in-google-directory-api?noredirect=1&lq=1
-            # Fortunately, when there are no custom OUs present, we won't
-            # need to check if a setting change was made at the top-level
-            # OU in the Rego; because no custom OUs have been created, any
-            # changes have to apply to the top-level OU.
+
+    def get_toplevel_ou(self, service) -> str:
+        '''
+        Gets the tenant name using the directory API
+
+        :param service: a directory_v1 service instance
+        '''
+
+        try:
+            response = service.orgunits().list(customerId='my_customer',
+                                        orgUnitPath='/',
+                                        type='children').execute()
+            # Because we set orgUnitPath to / and type to children, the API call will only
+            # return the second-level OUs, meaning the parentOrgUnitId of any of the OUs returned
+            # will point us to OU of the entire organization
+            if 'organizationUnits' not in response:
+                # No custom OUs have been created. In this case, we can't
+                # determine the name of the top-level OU. See:
+                # https://stackoverflow.com/questions/26936357/google-directory-api-org-name-of-root-org-unit-path
+                # https://stackoverflow.com/questions/60464432/cannot-get-root-orgunit-in-google-directory-api?noredirect=1&lq=1
+                # Fortunately, when there are no custom OUs present, we won't
+                # need to check if a setting change was made at the top-level
+                # OU in the Rego; because no custom OUs have been created, any
+                # changes have to apply to the top-level OU.
+                return ""
+            parent_ou = response['organizationUnits'][0]['parentOrgUnitId']
+            response = service.orgunits().get(customerId='my_customer', orgUnitPath=parent_ou).execute()
+            ou_name = response['name']
+            self.successful_calls.add("directory/v1/orgunits/list")
+            return ou_name
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting top level OU: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/orgunits/list")
             return ""
-        parent_ou = response['organizationUnits'][0]['parentOrgUnitId']
-        response = service.orgunits().get(customerId='my_customer', orgUnitPath=parent_ou).execute()
-        ou_name = response['name']
-        return ou_name
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting top level OU: {exc}",
-            RuntimeWarning
-        )
-        return ""
-
-
-def get_tenant_info(service) -> dict:
-    '''
-    Gets the high-level tenant info using the directory API
-
-    :param service: a directory_v1 service instance
-    '''
-    try:
-        response = service.customers().get(customerKey="my_customer").execute()
-        return {'id': response['id'],
-        'domain': response['customerDomain'],
-        'name': response['postalAddress']['organizationName'],
-        'topLevelOU': get_toplevel_ou(service)}
-    except Exception as exc:
-        warnings.warn(
-            f"An exception was thrown trying to get the tenant info: {exc}",
-            RuntimeWarning
-        )
-        return {'id': 'Error Retrieving',
-        'domain': 'Error Retrieving',
-        'name': 'Error Retrieving',
-        'topLevelOU': 'Error Retrieving'}
-
-
-def get_gws_logs(products: list, service, event: str) -> dict:
-    '''
-    Gets the GWS admin audit logs with the specified event name.
-    This function will also some parsing and filtering to ensure that an appropriate
-    log event is matched to the appropriate product.
-    This is to prevent the same log event from being duplicated
-    across products in the resulting provider JSON.
-
-    :param products: a narrowed list of the products being invoked
-    :param service: service is a Google reports API object, created from successfully
-    authenticating in auth.py
-    :param event: the name of the specific event we are querying for.
-    '''
-
-    # Filter responses by org_unit id
-    response = (service.activities().list(userKey='all',
-            applicationName='admin',
-            eventName=event).execute()).get('items', [])
-
-
-    # Used for filtering duplicate events
-    prod_to_app_name_values = {
-        'calendar': ['Calendar'],
-        'chat': ['Google Chat', 'Google Workspace Marketplace'],
-        'commoncontrols': [
-            'Security',
-            'Google Workspace Marketplace',
-            'Blogger',
-            'Google Cloud Platform Sharing Options',
-        ],
-        'drive': ['Drive and Docs'],
-        'gmail': ['Gmail'],
-        'groups': ['Groups for Business'],
-        'meet': ['Google Meet'],
-        'sites': ['Sites'],
-        'classroom': ['Classroom']
-    }
-    # create a subset of just the products we need from the dict above
-    subset_prod_to_app_name = {
-        prod: prod_to_app_name_values[prod]
-        for prod in products if prod in prod_to_app_name_values
-    }
-
-    products_to_logs = create_key_to_list(products)
-    # Certain events are not being currently being filtered because
-    # filtering for those events here would be duplicative of the Rego code
-    try:
-        # the value we want is nested several layers deep
-        # checks under the APPLICATION_NAME key for the correct app_name value
-        dup_events = (
-            'CHANGE_APPLICATION_SETTING',
-            'CREATE_APPLICATION_SETTING',
-            'DELETE_APPLICATION_SETTING'
+
+
+    def get_tenant_info(self, service) -> dict:
+        '''
+        Gets the high-level tenant info using the directory API
+
+        :param service: a directory_v1 service instance
+        '''
+        try:
+            response = service.customers().get(customerKey="my_customer").execute()
+            self.successful_calls.add("directory/v1/domains/list")
+            return {'id': response['id'],
+            'domain': response['customerDomain'],
+            'name': response['postalAddress']['organizationName'],
+            'topLevelOU': self.get_toplevel_ou(service)}
+        except Exception as exc:
+            warnings.warn(
+                f"An exception was thrown trying to get the tenant info: {exc}",
+                RuntimeWarning
             )
-        if event in dup_events:
-            app_name = 'APPLICATION_NAME'
-            for report in response:
-                for events in report['events']:
-                    parameters = events.get('parameters', [])
-                    for parameter in parameters:
-                        if parameter.get('name') == app_name:
-                            param_val = parameter.get('value')
-                            for prod, app_values in subset_prod_to_app_name.items():
-                                if param_val in app_values:
-                                    products_to_logs[prod].append(report)
-        else: # no filtering append entire response to relevant product
-            for prod in products:
-                products_to_logs[prod].extend(response)
-    except Exception as exc:
-        warnings.warn(
-            f"An exception was thrown while getting the logs; outputs will be incorrect: {exc}",
-            RuntimeWarning
-        )
-    return products_to_logs
-
-def get_group_settings(services) -> dict:
-    '''
-    Gets all of the group info using the directory API and group settings API
-
-    :param services: a service instance
-    '''
-
-    try:
-        # set up the services
+            self.unsuccessful_calls.add("directory/v1/domains/list")
+            return {'id': 'Error Retrieving',
+            'domain': 'Error Retrieving',
+            'name': 'Error Retrieving',
+            'topLevelOU': 'Error Retrieving'}
+
+
+    def get_gws_logs(self, products: list, service, event: str) -> dict:
+        '''
+        Gets the GWS admin audit logs with the specified event name.
+        This function will also some parsing and filtering to ensure that an appropriate
+        log event is matched to the appropriate product.
+        This is to prevent the same log event from being duplicated
+        across products in the resulting provider JSON.
+
+        :param products: a narrowed list of the products being invoked
+        :param service: service is a Google reports API object, created from successfully
+        authenticating in auth.py
+        :param event: the name of the specific event we are querying for.
+        '''
+
+        # Filter responses by org_unit id
+        response = (service.activities().list(userKey='all',
+                applicationName='admin',
+                eventName=event).execute()).get('items', [])
+
+
+        # Used for filtering duplicate events
+        prod_to_app_name_values = {
+            'calendar': ['Calendar'],
+            'chat': ['Google Chat', 'Google Workspace Marketplace'],
+            'commoncontrols': [
+                'Security',
+                'Google Workspace Marketplace',
+                'Blogger',
+                'Google Cloud Platform Sharing Options',
+            ],
+            'drive': ['Drive and Docs'],
+            'gmail': ['Gmail'],
+            'groups': ['Groups for Business'],
+            'meet': ['Google Meet'],
+            'sites': ['Sites'],
+            'classroom': ['Classroom']
+        }
+        # create a subset of just the products we need from the dict above
+        subset_prod_to_app_name = {
+            prod: prod_to_app_name_values[prod]
+            for prod in products if prod in prod_to_app_name_values
+        }
+
+        products_to_logs = create_key_to_list(products)
+        # Certain events are not being currently being filtered because
+        # filtering for those events here would be duplicative of the Rego code
+        try:
+            # the value we want is nested several layers deep
+            # checks under the APPLICATION_NAME key for the correct app_name value
+            dup_events = (
+                'CHANGE_APPLICATION_SETTING',
+                'CREATE_APPLICATION_SETTING',
+                'DELETE_APPLICATION_SETTING'
+                )
+            if event in dup_events:
+                app_name = 'APPLICATION_NAME'
+                for report in response:
+                    for events in report['events']:
+                        parameters = events.get('parameters', [])
+                        for parameter in parameters:
+                            if parameter.get('name') == app_name:
+                                param_val = parameter.get('value')
+                                for prod, app_values in subset_prod_to_app_name.items():
+                                    if param_val in app_values:
+                                        products_to_logs[prod].append(report)
+            else: # no filtering append entire response to relevant product
+                for prod in products:
+                    products_to_logs[prod].extend(response)
+        except Exception as exc:
+            warnings.warn(
+                f"An exception was thrown while getting the logs; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+        return products_to_logs
+
+    def get_group_settings(self, services) -> dict:
+        '''
+        Gets all of the group info using the directory API and group settings API
+
+        :param services: a service instance
+        '''
+
         group_service = services['groups']
         domain_service = services['directory']
-        # gather all of the domains within a suite to get groups
-        response = domain_service.domains().list(customer="my_customer").execute()
-        domains = {d['domainName'] for d in response['domains']}
+        try:
+            # gather all of the domains within a suite to get groups
+            response = domain_service.domains().list(customer="my_customer").execute()
+            domains = {d['domainName'] for d in response['domains']}
+            self.successful_calls.add("directory/v1/domains/list")
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting group settings; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/domains/list")
+            return {'group_settings': []}
+
+        try:
+            # get the group settings for each groups
+            group_settings = []
+            for domain in domains:
+                response = domain_service.groups().list(domain=domain).execute()
+                for group in response.get('groups'):
+                    email = group.get('email')
+                    group_settings.append(group_service.groups().get(groupUniqueId=email).execute())
+            self.successful_calls.add("groups-settings/v1/groups/get")
+            return {'group_settings': group_settings}
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting group settings; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("groups-settings/v1/groups/get")
+            return {'group_settings': []}
+
+    def call_gws_providers(self, products: list, services, quiet) -> dict:
+        '''
+        Calls the relevant GWS APIs to get the data we need for the baselines.
+        Data such as the admin audit log, super admin users etc.
+
+        :param products: list of product names to check
+        :param services: a dict of service objects.
+        :param quiet: suppress tqdm output
+        service is a Google reports API object, created from successfully authenticating in auth.py
+        '''
+        # create a inverse dictionary containing a mapping of event => list of products
+        events_to_products = create_subset_inverted_dict(EVENTS, products)
+        events_to_products_bar = tqdm(events_to_products.items(), leave=False, disable=quiet)
+
+        # main aggregator dict
+        product_to_logs = create_key_to_list(products)
+        product_to_items = {}
+        ou_ids = set()
+        ou_ids.add("") # certain settings have no OU
+        try:
+            # Add top level organization unit name
+            ou_ids.add(self.get_toplevel_ou(services['directory']))
+            # get all organizational unit data
+            product_to_items['organizational_units'] = self.get_ous(services['directory'])
+            for orgunit in product_to_items['organizational_units']['organizationUnits']:
+                ou_ids.add(orgunit['name'])
+            # add just organizational unit names to a field]
+            product_to_items['organizational_unit_names'] = list(ou_ids)
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting tenant data: {exc}",
+                RuntimeWarning
+            )
 
-        # get the group settings for each groups
-        group_settings = []
-        for domain in domains:
-            response = domain_service.groups().list(domain=domain).execute()
-            for group in response.get('groups'):
-                email = group.get('email')
-                group_settings.append(group_service.groups().get(groupUniqueId=email).execute())
-        return {'group_settings': group_settings}
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting group settings; outputs will be incorrect: {exc}",
-            RuntimeWarning
-        )
-        return {'group_settings': []}
-
-def call_gws_providers(products: list, services, quiet) -> dict:
-    '''
-    Calls the relevant GWS APIs to get the data we need for the baselines.
-    Data such as the admin audit log, super admin users etc.
-
-    :param products: list of product names to check
-    :param services: a dict of service objects.
-    :param quiet: suppress tqdm output
-    service is a Google reports API object, created from successfully authenticating in auth.py
-    '''
-    # create a inverse dictionary containing a mapping of event => list of products
-    events_to_products = create_subset_inverted_dict(EVENTS, products)
-    events_to_products_bar = tqdm(events_to_products.items(), leave=False, disable=quiet)
-
-    # main aggregator dict
-    product_to_logs = create_key_to_list(products)
-    product_to_items = {}
-    ou_ids = set()
-    ou_ids.add("") # certain settings have no OU
-    try:
-        # Add top level organization unit name
-        ou_ids.add(get_toplevel_ou(services['directory']))
-        # get all organizational unit data
-        product_to_items['organizational_units'] = get_ous(services['directory'])
-        for orgunit in product_to_items['organizational_units']['organizationUnits']:
-            ou_ids.add(orgunit['name'])
-        # add just organizational unit names to a field]
-        product_to_items['organizational_unit_names'] = list(ou_ids)
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting tenant data: {exc}",
-            RuntimeWarning
-        )
-
-    # call the api once per event type
-    try:
-        for event, product_list in events_to_products_bar:
-            products = ', '.join(product_list)
-            bar_descr = f"Running Provider: Exporting {event} events for {products}..."
-            events_to_products_bar.set_description(bar_descr)
-
-            # gets the GWS admin audit logs and merges them into product_to_logs
-            # aggregator dict
-            product_to_logs = merge_dicts(
-                product_to_logs,
-                get_gws_logs(
-                    products=product_list,
-                    service=services['reports'],
-                    event=event
+        # call the api once per event type
+        try:
+            for event, product_list in events_to_products_bar:
+                products = ', '.join(product_list)
+                bar_descr = f"Running Provider: Exporting {event} events for {products}..."
+                events_to_products_bar.set_description(bar_descr)
+
+                # gets the GWS admin audit logs and merges them into product_to_logs
+                # aggregator dict
+                product_to_logs = merge_dicts(
+                    product_to_logs,
+                    self.get_gws_logs(
+                        products=product_list,
+                        service=services['reports'],
+                        event=event
+                    )
                 )
+                self.successful_calls.add("reports/v1/activity/list")
+        except Exception as exc:
+            warnings.warn(
+                f"Provider Exception thrown while getting the logs; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("reports/v1/activity/list")
+
+        # repacks the main aggregator into the original form
+        # that the api returns the data in; under an 'items' key.
+        # Then we put this key under a {product}_log key for the Rego code
+        try:
+            for product, logs in product_to_logs.items():
+                key_name = f"{product}_logs"
+                product_to_items[key_name] = {'items': logs}
+
+            # get tenant metadata for report front page header
+            product_to_items['tenant_info'] = self.get_tenant_info(services['directory'])
+
+            if 'gmail' in product_to_logs: # add dns info if gmail is being run
+                product_to_items.update(self.get_dnsinfo(services['directory']))
+
+            if 'commoncontrols' in product_to_logs: # add list of super admins if CC is being run
+                product_to_items.update(self.get_super_admins(services['directory']))
+
+            if 'groups' in product_to_logs:
+                product_to_items.update(self.get_group_settings(services=services))
+
+        except Exception as exc:
+            warnings.warn(
+                f"Uncaught Exception thrown while getting other data: {exc}",
+                RuntimeWarning
             )
-    except Exception as exc:
-        warnings.warn(
-            f"Provider Exception thrown while getting the logs; outputs will be incorrect: {exc}",
-            RuntimeWarning
-        )
-
-    # repacks the main aggregator into the original form
-    # that the api returns the data in; under an 'items' key.
-    # Then we put this key under a {product}_log key for the Rego code
-    try:
-        for product, logs in product_to_logs.items():
-            key_name = f"{product}_logs"
-            product_to_items[key_name] = {'items': logs}
-
-        # get tenant metadata for report front page header
-        product_to_items['tenant_info'] = get_tenant_info(services['directory'])
-
-        if 'gmail' in product_to_logs: # add dns info if gmail is being run
-            product_to_items.update(get_dnsinfo(services['directory']))
-
-        if 'commoncontrols' in product_to_logs: # add list of super admins if CC is being run
-            product_to_items.update(get_super_admins(services['directory']))
-
-        if 'groups' in product_to_logs:
-            product_to_items.update(get_group_settings(services=services))
-
-    except Exception as exc:
-        warnings.warn(
-            f"Uncaught Exception thrown while getting other data: {exc}",
-            RuntimeWarning
-        )
-    return product_to_items
+        return product_to_items
diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index 57065b7f..6d62f8d8 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -9,6 +9,28 @@
 import pandas as pd
 from scubagoggles.utils import rel_abs_path
 
+def get_reference_a_tag(api_call : str) -> str:
+    '''
+    Craft the link to the documentation page for each API call.
+    
+    :param api_call: a string representing the API call, such as "directory/v1/users/list".
+    '''
+    api = api_call.split('/')[0]
+    call = '/'.join(api_call.split('/')[1:])
+    # All APIs except for the groups-settings api have "rest/" after "reference/"
+    api_type = "" if api == "groups-settings" else "rest/"
+    return f'<a href="https://developers.google.com/admin-sdk/{api}/reference/{api_type}{call}"> \
+        {api_call}</a>'
+
+API_LINKS = {api_call: get_reference_a_tag(api_call) for api_call in [
+    "directory/v1/users/list",
+    "directory/v1/orgunits/list",
+    "directory/v1/domains/list",
+    "directory/v1/groups/list",
+    "reports/v1/activities/list",
+    "group-settings/v1/groups/get"
+]}
+
 def get_test_result(requirement_met : bool, criticality : str, no_such_events : bool) -> str:
     '''
     Checks the Rego to see if the baseline passed or failed and indicates the criticality
@@ -141,7 +163,8 @@ def build_report_html(fragments : list, product : str,
     return html
 
 def rego_json_to_html(test_results_data : str, product : list, out_path : str,
-tenant_name : str, main_report_name : str, prod_to_fullname: dict, product_policies) -> None:
+tenant_name : str, main_report_name : str, prod_to_fullname: dict, product_policies,
+successful_calls : set, unsuccessful_calls : set) -> None:
     '''
     Transforms the Rego JSON output into HTML
 
@@ -152,6 +175,8 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str,
     :param main_report_name: report_name: Name of the main report HTML file.
     :param prod_to_fullname: dict containing mapping of the product full names
     :param product_policies: dict containing policies read from the baseline markdown
+    :param successful_calls: set with the set of successful calls
+    :param unsuccessful_calls: set with the set of unsuccessful calls
     '''
 
     product_capitalized = product.capitalize()
@@ -164,7 +189,8 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str,
         "Warning": 0,
         "Fail": 0,
         "N/A": 0,
-        "No events found": 0
+        "No events found": 0,
+        "Error": 0
     }
 
     for baseline_group in product_policies:
@@ -172,48 +198,83 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str,
         for control in baseline_group['Controls']:
             tests = [test for test in test_results_data if test['PolicyId'] == control['Id']]
             for test in tests:
-                result = get_test_result(test['RequirementMet'], test['Criticality'],
-                test['NoSuchEvent'])
-                report_stats[result] = report_stats[result] + 1
-                details = test['ReportDetails']
-
-                if result == "No events found":
-                    warning_icon = "<object data='./images/triangle-exclamation-solid.svg'\
-                        width='15'\
-                        height='15'>\
-                        </object>"
-                    details = warning_icon + " " + test['ReportDetails']
-
-                # As rules doesn't have it's own baseline, Rules and Common Controls
-                # need to be handled specially
-                if product_capitalized == "Rules":
-                    if 'Not-Implemented' in test['Criticality']:
-                        # The easiest way to identify the GWS.COMMONCONTROLS.14.1v1
+                # The following if/else makes the error handling backwards compatible until
+                # all Regos are updated.
+                if 'Prerequisites' not in test:
+                    prereqs = set()
+                else:
+                    prereqs = set(test['Prerequisites'])
+                # A call is failed if it is either missing from the successful_calls set
+                # or present in the unsuccessful_calls
+                failed_calls = set().union(
+                    prereqs.difference(successful_calls),
+                    prereqs.intersection(unsuccessful_calls)
+                )
+                if len(failed_calls) > 0:
+                    result = "Error"
+                    report_stats["Error"] += 1
+                    failed_api_links = [API_LINKS[api] for api in failed_calls if api in API_LINKS]
+                    failed_functions = [call for call in failed_calls if call not in API_LINKS]
+                    failed_details = ""
+                    if len(failed_api_links) > 0:
+                        links = ', '.join(failed_api_links)
+                        failed_details += f"This test depends on the following API call(s) " \
+                            f"which did not execute successfully: {links}. " \
+                            "See terminal output for more details. "
+                    if len(failed_functions) > 0:
+                        failed_details += f"This test depends on the following function(s) " \
+                            f"which did not execute successfully: {', '.join(failed_functions)}." \
+                            "See terminal output for more details."
+                    table_data.append({
+                        'Control ID': control['Id'],
+                        'Requirement': control['Value'],
+                        'Result': "Error",
+                        'Criticality': test['Criticality'],
+                        'Details': failed_details
+                    })
+                else:
+                    result = get_test_result(test['RequirementMet'], test['Criticality'],
+                    test['NoSuchEvent'])
+                    report_stats[result] = report_stats[result] + 1
+                    details = test['ReportDetails']
+
+                    if result == "No events found":
+                        warning_icon = "<object data='./images/triangle-exclamation-solid.svg'\
+                            width='15'\
+                            height='15'>\
+                            </object>"
+                        details = warning_icon + " " + test['ReportDetails']
+
+                    # As rules doesn't have it's own baseline, Rules and Common Controls
+                    # need to be handled specially
+                    if product_capitalized == "Rules":
+                        if 'Not-Implemented' in test['Criticality']:
+                            # The easiest way to identify the GWS.COMMONCONTROLS.14.1v1
+                            # results that belong to the Common Controls report is they're
+                            # marked as Not-Implemented. This if excludes them from the
+                            # rules report.
+                            continue
+                        table_data.append({
+                            'Control ID': control['Id'],
+                            'Rule Name': test['Requirement'],
+                            'Result': result,
+                            'Criticality': test['Criticality'],
+                            'Rule Description': test['ReportDetails']})
+                    elif product_capitalized == "Commoncontrols" \
+                        and baseline_group['GroupName'] == 'System-defined Rules' \
+                        and 'Not-Implemented' not in test['Criticality']:
+                        # The easiest way to identify the System-defined Rules
                         # results that belong to the Common Controls report is they're
-                        # marked as Not-Implemented. This if excludes them from the
-                        # rules report.
+                        # marked as Not-Implemented. This if excludes the full results
+                        # from the Common Controls report.
                         continue
-                    table_data.append({
+                    else:
+                        table_data.append({
                         'Control ID': control['Id'],
-                        'Rule Name': test['Requirement'],
+                        'Requirement': control['Value'],
                         'Result': result,
                         'Criticality': test['Criticality'],
-                        'Rule Description': test['ReportDetails']})
-                elif product_capitalized == "Commoncontrols" \
-                    and baseline_group['GroupName'] == 'System-defined Rules' \
-                    and 'Not-Implemented' not in test['Criticality']:
-                    # The easiest way to identify the System-defined Rules
-                    # results that belong to the Common Controls report is they're
-                    # marked as Not-Implemented. This if excludes the full results
-                    # from the Common Controls report.
-                    continue
-                else:
-                    table_data.append({
-                    'Control ID': control['Id'],
-                    'Requirement': control['Value'],
-                    'Result': result,
-                    'Criticality': test['Criticality'],
-                    'Details': details})
+                        'Details': details})
         fragments.append(f"<h2>{product_upper}-{baseline_group['GroupNumber']} \
         {baseline_group['GroupName']}</h2>")
         fragments.append(create_html_table(table_data))

From 4b0641ec6862759f60ae16df2cfa108d04e6ae71 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Wed, 24 Jan 2024 17:50:59 -0800
Subject: [PATCH 09/35] Merge latest changes from main

---
 rego/Commoncontrols.rego          |   1 +
 scubagoggles/orchestrator.py      |  20 +-
 scubagoggles/provider.py          | 852 ++++++++++++++++--------------
 scubagoggles/reporter/reporter.py | 139 +++--
 4 files changed, 558 insertions(+), 454 deletions(-)

diff --git a/rego/Commoncontrols.rego b/rego/Commoncontrols.rego
index db577ac7..df2d1278 100644
--- a/rego/Commoncontrols.rego
+++ b/rego/Commoncontrols.rego
@@ -713,6 +713,7 @@ tests contains {
 #--
 tests contains {
     "PolicyId": "GWS.COMMONCONTROLS.7.1v0.1",
+    "Prerequisites": ["directory/v1/users/list"],
     "Criticality": "Shall",
     "ReportDetails": concat("", [
         concat("", ["The following super admins are configured: ", concat(", ", SuperAdmins)]),
diff --git a/scubagoggles/orchestrator.py b/scubagoggles/orchestrator.py
index ce6c4407..b4cc3a44 100644
--- a/scubagoggles/orchestrator.py
+++ b/scubagoggles/orchestrator.py
@@ -12,7 +12,7 @@
 from googleapiclient.discovery import build
 
 from scubagoggles.auth import gws_auth
-from scubagoggles.provider import call_gws_providers
+from scubagoggles.provider import Provider
 from scubagoggles.run_rego import opa_eval
 from scubagoggles.reporter import reporter, md_parser
 from scubagoggles.utils import rel_abs_path
@@ -63,8 +63,10 @@ def run_gws_providers(args, services):
     out_folder = args.outputpath
     provider_dict = {}
 
-    provider_dict = call_gws_providers(products, services, args.quiet)
-
+    provider = Provider()
+    provider_dict = provider.call_gws_providers(products, services, args.quiet)
+    provider_dict['successful_calls'] = list(provider.successful_calls)
+    provider_dict['unsuccessful_calls'] = list(provider.unsuccessful_calls)
     settings_json = json.dumps(provider_dict, indent = 4)
     out_path = out_folder + f'/{args.outputproviderfilename}.json'
     with open(out_path, mode="w", encoding='UTF-8') as outfile:
@@ -146,8 +148,14 @@ def run_reporter(args):
     with open(test_results_json, mode='r', encoding='UTF-8') as file:
         test_results_data = json.load(file)
 
-    # baseline_path
+    # Get the successful/unsuccessful commands
+    settings_name = f'{out_folder}/{args.outputproviderfilename}.json'
+    with open(settings_name, mode='r', encoding='UTF-8') as file:
+        settings_data = json.load(file)
+    successful_calls = set(settings_data['successful_calls'])
+    unsuccessful_calls = set(settings_data['unsuccessful_calls'])
 
+    # baseline_path
     subset_prod_to_fullname = {
         key: prod_to_fullname[key]
         for key in args.baselines
@@ -188,7 +196,9 @@ def run_reporter(args):
             tenant_domain,
             main_report_name,
             prod_to_fullname,
-            baseline_policies[product]
+            baseline_policies[product],
+            successful_calls,
+            unsuccessful_calls
         )
 
     # Make the report front page
diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index 7e765c08..1a859dc3 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -86,428 +86,460 @@
 
 DNSClient = RobustDNSClient()
 
-def get_spf_records(domains: list) -> list:
-    '''
-    Gets the SPF records for each domain in domains.
-
-    :param domains: The list of domain names (strings).
-    '''
-    results = []
-    n_low_confidence = 0
-    for domain in domains:
-        result = DNSClient.query(domain)
-        if not result['HighConfidence']:
-            n_low_confidence += 1
-        results.append({
-            "domain": domain,
-            "rdata": result["Answers"],
-            "log": result["LogEntries"]
-        })
-    if n_low_confidence > 0:
-        warnings.warn(f"get_spf_records: for {n_low_confidence} domain(s), \
-the traditional DNS queries returned an empty answer \
-section and the DoH queries failed. Will assume SPF not configured, but \
-can't guarantee that failure isn't due to something like split horizon DNS. \
-See ProviderSettingsExport.json under 'spf_records' for more details.", RuntimeWarning)
-    return results
-
-def get_dkim_records(domains : list) -> list:
-    '''
-    Gets the DKIM records for each domain in domains.
-
-    :param domains: The list of domain names (strings).
-    '''
-    results = []
-    n_low_confidence = 0
-    for domain in domains:
-        qnames = [f"{selector}._domainkey.{domain}" for selector in selectors]
-        log_entries = []
-        for qname in qnames:
+class Provider:
+    def __init__(self):
+        self.successful_calls = set()
+        self.unsuccessful_calls = set()
+
+    def get_spf_records(self, domains: list) -> list:
+        '''
+        Gets the SPF records for each domain in domains.
+
+        :param domains: The list of domain names (strings).
+        '''
+        results = []
+        n_low_confidence = 0
+        for domain in domains:
+            result = DNSClient.query(domain)
+            if not result['HighConfidence']:
+                n_low_confidence += 1
+            results.append({
+                "domain": domain,
+                "rdata": result["Answers"],
+                "log": result["LogEntries"]
+            })
+        if n_low_confidence > 0:
+            warnings.warn(f"get_spf_records: for {n_low_confidence} domain(s), \
+    the traditional DNS queries returned an empty answer \
+    section and the DoH queries failed. Will assume SPF not configured, but \
+    can't guarantee that failure isn't due to something like split horizon DNS. \
+    See ProviderSettingsExport.json under 'spf_records' for more details.", RuntimeWarning)
+        return results
+
+    def get_dkim_records(self, domains : list) -> list:
+        '''
+        Gets the DKIM records for each domain in domains.
+
+        :param domains: The list of domain names (strings).
+        '''
+        results = []
+        n_low_confidence = 0
+        for domain in domains:
+            qnames = [f"{selector}._domainkey.{domain}" for selector in selectors]
+            log_entries = []
+            for qname in qnames:
+                result = DNSClient.query(qname)
+                log_entries.extend(result['LogEntries'])
+                if len(result['Answers']) == 0:
+                    # The DKIM record does not exist with this selector, we need to try again with
+                    # a different one
+                    continue
+                # Otherwise, the DKIM record exists with this selector, no need to try the rest
+                break
+
+            if not result['HighConfidence']:
+                n_low_confidence += 1
+            results.append({
+                "domain": domain,
+                "rdata": result["Answers"],
+                "log": log_entries
+            })
+
+        if n_low_confidence > 0:
+            warnings.warn(f"get_dkim_records: for {n_low_confidence} domain(s), \
+    the traditional DNS queries returned an empty answer \
+    section and the DoH queries failed. Will assume DKIM not configured, but \
+    can't guarantee that failure isn't due to something like split horizon DNS. \
+    See ProviderSettingsExport.json under 'dkim_records' for more details.", RuntimeWarning)
+        return results
+
+    def get_dmarc_records(self, domains : list) -> list:
+        '''
+        Gets the DMARC records for each domain in domains.
+
+        :param domains: The list of domain names (strings).
+        '''
+        results = []
+        n_low_confidence = 0
+        for domain in domains:
+            log_entries = []
+            qname = f"_dmarc.{domain}"
             result = DNSClient.query(qname)
             log_entries.extend(result['LogEntries'])
-            if len(result['Answers']) == 0:
-                # The DKIM record does not exist with this selector, we need to try again with
-                # a different one
-                continue
-            # Otherwise, the DKIM record exists with this selector, no need to try the rest
-            break
-
-        if not result['HighConfidence']:
-            n_low_confidence += 1
-        results.append({
-            "domain": domain,
-            "rdata": result["Answers"],
-            "log": log_entries
-        })
-
-    if n_low_confidence > 0:
-        warnings.warn(f"get_dkim_records: for {n_low_confidence} domain(s), \
-the traditional DNS queries returned an empty answer \
-section and the DoH queries failed. Will assume DKIM not configured, but \
-can't guarantee that failure isn't due to something like split horizon DNS. \
-See ProviderSettingsExport.json under 'dkim_records' for more details.", RuntimeWarning)
-    return results
-
-def get_dmarc_records(domains : list) -> list:
-    '''
-    Gets the DMARC records for each domain in domains.
-
-    :param domains: The list of domain names (strings).
-    '''
-    results = []
-    n_low_confidence = 0
-    for domain in domains:
-        log_entries = []
-        qname = f"_dmarc.{domain}"
-        result = DNSClient.query(qname)
-        log_entries.extend(result['LogEntries'])
-        if len(result["Answers"]) == 0:
-            # The domain does not exist. If the record is not available at the full domain
-            # level, we need to check at the organizational domain level.
-            labels = domain.split(".")
-            org_domain = f"{labels[-2]}.{labels[-1]}"
-            result = DNSClient.query(f"_dmarc.{org_domain}")
-            log_entries.extend(result['LogEntries'])
-        if not result['HighConfidence']:
-            n_low_confidence += 1
-        results.append({
-            "domain": domain,
-            "rdata": result["Answers"],
-            "log": log_entries
-        })
-    if n_low_confidence > 0:
-        warnings.warn(f"get_dmarc_records: for {n_low_confidence} domain(s), \
-the traditional DNS queries returned an empty answer \
-section and the DoH queries failed. Will assume DMARC not configured, but \
-can't guarantee that failure isn't due to something like split horizon DNS. \
-See ProviderSettingsExport.json under 'dmarc_records' for more details.", RuntimeWarning)
-    return results
-
-def get_dnsinfo(service):
-    '''
-    Gets DNS Information for Gmail baseline
-
-    :param service: a directory_v1 service instance
-    '''
-    output = {"domains": [], "spf_records": [], "dkim_records": [], "dmarc_records": []}
-
-    # Determine the tenant's domains via the API
-    response = service.domains().list(customer="my_customer").execute()
-    domains = {d['domainName'] for d in response['domains']}
-
-    if len(domains) == 0:
-        warnings.warn("No domains found.", RuntimeWarning)
+            if len(result["Answers"]) == 0:
+                # The domain does not exist. If the record is not available at the full domain
+                # level, we need to check at the organizational domain level.
+                labels = domain.split(".")
+                org_domain = f"{labels[-2]}.{labels[-1]}"
+                result = DNSClient.query(f"_dmarc.{org_domain}")
+                log_entries.extend(result['LogEntries'])
+            if not result['HighConfidence']:
+                n_low_confidence += 1
+            results.append({
+                "domain": domain,
+                "rdata": result["Answers"],
+                "log": log_entries
+            })
+        if n_low_confidence > 0:
+            warnings.warn(f"get_dmarc_records: for {n_low_confidence} domain(s), \
+    the traditional DNS queries returned an empty answer \
+    section and the DoH queries failed. Will assume DMARC not configured, but \
+    can't guarantee that failure isn't due to something like split horizon DNS. \
+    See ProviderSettingsExport.json under 'dmarc_records' for more details.", RuntimeWarning)
+        return results
+
+    def get_dnsinfo(self, service):
+        '''
+        Gets DNS Information for Gmail baseline
+
+        :param service: a directory_v1 service instance
+        '''
+        output = {"domains": [], "spf_records": [], "dkim_records": [], "dmarc_records": []}
+
+        # Determine the tenant's domains via the API
+        response = service.domains().list(customer="my_customer").execute()
+        domains = {d['domainName'] for d in response['domains']}
+
+        if len(domains) == 0:
+            warnings.warn("No domains found.", RuntimeWarning)
+            return output
+
+        output["domains"].extend(domains)
+
+        try:
+            output["spf_records"] = get_spf_records(domains)
+            self.successful_calls.add("get_spf_records")
+        except Exception as exc:
+            output["spf_records"] = []
+            warnings.warn(f"An exception was thrown by get_spf_records: {exc}", RuntimeWarning)
+            self.unsuccessful_calls.add("get_spf_records")
+        try:
+            output["dkim_records"] = get_dkim_records(domains)
+            self.successful_calls.add("get_dkim_records")
+        except Exception as exc:
+            output["dkim_records"] = []
+            warnings.warn(f"An exception was thrown by get_dkim_records: {exc}", RuntimeWarning)
+            self.unsuccessful_calls.add("get_dkim_records")
+        try:
+            output["dmarc_records"] = get_dmarc_records(domains)
+            self.successful_calls.add("get_dmarc_records")
+        except Exception as exc:
+            output["dmarc_records"] = []
+            warnings.warn(f"An exception was thrown by get_dmarc_records: {exc}", RuntimeWarning)
+            self.unsuccessful_calls.add("get_dmarc_records")
         return output
 
-    output["domains"].extend(domains)
-
-    try:
-        output["spf_records"] = get_spf_records(domains)
-    except Exception as exc:
-        output["spf_records"] = []
-        warnings.warn(f"An exception was thrown by get_spf_records: {exc}", RuntimeWarning)
-    try:
-        output["dkim_records"] = get_dkim_records(domains)
-    except Exception as exc:
-        output["dkim_records"] = []
-        warnings.warn(f"An exception was thrown by get_dkim_records: {exc}", RuntimeWarning)
-    try:
-        output["dmarc_records"] = get_dmarc_records(domains)
-    except Exception as exc:
-        output["dmarc_records"] = []
-        warnings.warn(f"An exception was thrown by get_dmarc_records: {exc}", RuntimeWarning)
-    return output
-
-def get_super_admins(service) -> dict:
-    '''
-    Gets the org unit/primary email of all super admins, using the directory API
-
-    :param service: a directory_v1 service instance
-    '''
-    try:
-        response = service.users().list(customer="my_customer", query="isAdmin=True").execute()
-        admins = []
-        for user in response['users']:
-            org_unit = user['orgUnitPath']
-            # strip out the leading '/'
-            org_unit = org_unit[1:] if org_unit.startswith('/') else org_unit
-            email = user['primaryEmail']
-            admins.append({'primaryEmail': email, 'orgUnitPath': org_unit})
-        return {'super_admins': admins}
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting super admins; outputs will be incorrect: {exc}",
-            RuntimeWarning
-        )
-        return {'super_admins': []}
-
-def get_ous(service) -> dict:
-    '''
-    Gets the organizational units using the directory API
-
-    :param service: a directory_v1 service instance
-    '''
-
-    try:
-        response = service.orgunits().list(customerId='my_customer').execute()
-        if 'organizationUnits' not in response:
+    def get_super_admins(self, service) -> dict:
+        '''
+        Gets the org unit/primary email of all super admins, using the directory API
+
+        :param service: a directory_v1 service instance
+        '''
+        try:
+            x/0
+            response = service.users().list(customer="my_customer", query="isAdmin=True").execute()
+            admins = []
+            for user in response['users']:
+                org_unit = user['orgUnitPath']
+                # strip out the leading '/'
+                org_unit = org_unit[1:] if org_unit.startswith('/') else org_unit
+                email = user['primaryEmail']
+                admins.append({'primaryEmail': email, 'orgUnitPath': org_unit})
+            self.successful_calls.add("directory/v1/users/list")
+            return {'super_admins': admins}
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting super admins; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/users/list")
+            return {'super_admins': []}
+
+    def get_ous(self, service) -> dict:
+        '''
+        Gets the organizational units using the directory API
+
+        :param service: a directory_v1 service instance
+        '''
+
+        try:
+            response = service.orgunits().list(customerId='my_customer').execute()
+            self.successful_calls.add("directory/v1/orgunits/list")
+            if 'organizationUnits' not in response:
+                return {}
+            return response
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting top level OU: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/orgunits/list")
             return {}
-        return response
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting top level OU: {exc}",
-            RuntimeWarning
-        )
-        return {}
-
-def get_toplevel_ou(service) -> str:
-    '''
-    Gets the tenant name using the directory API
-
-    :param service: a directory_v1 service instance
-    '''
-
-    try:
-        response = service.orgunits().list(customerId='my_customer',
-                                    orgUnitPath='/',
-                                    type='children').execute()
-        # Because we set orgUnitPath to / and type to children, the API call will only
-        # return the second-level OUs, meaning the parentOrgUnitId of any of the OUs returned
-        # will point us to OU of the entire organization
-        if 'organizationUnits' not in response:
-            # No custom OUs have been created. In this case, we can't
-            # determine the name of the top-level OU. See:
-            # https://stackoverflow.com/questions/26936357/google-directory-api-org-name-of-root-org-unit-path
-            # https://stackoverflow.com/questions/60464432/cannot-get-root-orgunit-in-google-directory-api?noredirect=1&lq=1
-            # Fortunately, when there are no custom OUs present, we won't
-            # need to check if a setting change was made at the top-level
-            # OU in the Rego; because no custom OUs have been created, any
-            # changes have to apply to the top-level OU.
+
+    def get_toplevel_ou(self, service) -> str:
+        '''
+        Gets the tenant name using the directory API
+
+        :param service: a directory_v1 service instance
+        '''
+
+        try:
+            response = service.orgunits().list(customerId='my_customer',
+                                        orgUnitPath='/',
+                                        type='children').execute()
+            # Because we set orgUnitPath to / and type to children, the API call will only
+            # return the second-level OUs, meaning the parentOrgUnitId of any of the OUs returned
+            # will point us to OU of the entire organization
+            if 'organizationUnits' not in response:
+                # No custom OUs have been created. In this case, we can't
+                # determine the name of the top-level OU. See:
+                # https://stackoverflow.com/questions/26936357/google-directory-api-org-name-of-root-org-unit-path
+                # https://stackoverflow.com/questions/60464432/cannot-get-root-orgunit-in-google-directory-api?noredirect=1&lq=1
+                # Fortunately, when there are no custom OUs present, we won't
+                # need to check if a setting change was made at the top-level
+                # OU in the Rego; because no custom OUs have been created, any
+                # changes have to apply to the top-level OU.
+                return ""
+            parent_ou = response['organizationUnits'][0]['parentOrgUnitId']
+            response = service.orgunits().get(customerId='my_customer', orgUnitPath=parent_ou).execute()
+            ou_name = response['name']
+            self.successful_calls.add("directory/v1/orgunits/list")
+            return ou_name
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting top level OU: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/orgunits/list")
             return ""
-        parent_ou = response['organizationUnits'][0]['parentOrgUnitId']
-        response = service.orgunits().get(customerId='my_customer', orgUnitPath=parent_ou).execute()
-        ou_name = response['name']
-        return ou_name
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting top level OU: {exc}",
-            RuntimeWarning
-        )
-        return ""
-
-
-def get_tenant_info(service) -> dict:
-    '''
-    Gets the high-level tenant info using the directory API
-
-    :param service: a directory_v1 service instance
-    '''
-    try:
-        response = service.domains().list(customer="my_customer").execute()
-        primary_domain = ""
-        for domain in response['domains']:
-            if domain['isPrimary']:
-                primary_domain = domain['domainName']
-        return {
-            'domain': primary_domain,
-            'topLevelOU': get_toplevel_ou(service)
+
+
+    def get_tenant_info(self, service) -> dict:
+        '''
+        Gets the high-level tenant info using the directory API
+
+        :param service: a directory_v1 service instance
+        '''
+        try:
+            response = service.domains().list(customer="my_customer").execute()
+            self.successful_calls.add("directory/v1/domains/list")
+            primary_domain = ""
+            for domain in response['domains']:
+                if domain['isPrimary']:
+                    primary_domain = domain['domainName']
+            return {
+                'domain': primary_domain,
+                'topLevelOU': self.get_toplevel_ou(service)
+            }
+        except Exception as exc:
+            warnings.warn(
+                f"An exception was thrown trying to get the tenant info: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/domains/list")
+            return {
+                'domain': 'Error Retrieving',
+                'topLevelOU': 'Error Retrieving'
+            }
+
+
+    def get_gws_logs(self, products: list, service, event: str) -> dict:
+        '''
+        Gets the GWS admin audit logs with the specified event name.
+        This function will also some parsing and filtering to ensure that an appropriate
+        log event is matched to the appropriate product.
+        This is to prevent the same log event from being duplicated
+        across products in the resulting provider JSON.
+
+        :param products: a narrowed list of the products being invoked
+        :param service: service is a Google reports API object, created from successfully
+        authenticating in auth.py
+        :param event: the name of the specific event we are querying for.
+        '''
+
+        # Filter responses by org_unit id
+        response = (service.activities().list(userKey='all',
+                applicationName='admin',
+                eventName=event).execute()).get('items', [])
+
+
+        # Used for filtering duplicate events
+        prod_to_app_name_values = {
+            'calendar': ['Calendar'],
+            'chat': ['Google Chat', 'Google Workspace Marketplace'],
+            'commoncontrols': [
+                'Security',
+                'Google Workspace Marketplace',
+                'Blogger',
+                'Google Cloud Platform Sharing Options',
+            ],
+            'drive': ['Drive and Docs'],
+            'gmail': ['Gmail'],
+            'groups': ['Groups for Business'],
+            'meet': ['Google Meet'],
+            'sites': ['Sites'],
+            'classroom': ['Classroom']
         }
-    except Exception as exc:
-        warnings.warn(
-            f"An exception was thrown trying to get the tenant info: {exc}",
-            RuntimeWarning
-        )
-        return {
-            'domain': 'Error Retrieving',
-            'topLevelOU': 'Error Retrieving'
+        # create a subset of just the products we need from the dict above
+        subset_prod_to_app_name = {
+            prod: prod_to_app_name_values[prod]
+            for prod in products if prod in prod_to_app_name_values
         }
 
-
-def get_gws_logs(products: list, service, event: str) -> dict:
-    '''
-    Gets the GWS admin audit logs with the specified event name.
-    This function will also some parsing and filtering to ensure that an appropriate
-    log event is matched to the appropriate product.
-    This is to prevent the same log event from being duplicated
-    across products in the resulting provider JSON.
-
-    :param products: a narrowed list of the products being invoked
-    :param service: service is a Google reports API object, created from successfully
-    authenticating in auth.py
-    :param event: the name of the specific event we are querying for.
-    '''
-
-    # Filter responses by org_unit id
-    response = (service.activities().list(userKey='all',
-            applicationName='admin',
-            eventName=event).execute()).get('items', [])
-
-
-    # Used for filtering duplicate events
-    prod_to_app_name_values = {
-        'calendar': ['Calendar'],
-        'chat': ['Google Chat', 'Google Workspace Marketplace'],
-        'commoncontrols': [
-            'Security',
-            'Google Workspace Marketplace',
-            'Blogger',
-            'Google Cloud Platform Sharing Options',
-        ],
-        'drive': ['Drive and Docs'],
-        'gmail': ['Gmail'],
-        'groups': ['Groups for Business'],
-        'meet': ['Google Meet'],
-        'sites': ['Sites'],
-        'classroom': ['Classroom']
-    }
-    # create a subset of just the products we need from the dict above
-    subset_prod_to_app_name = {
-        prod: prod_to_app_name_values[prod]
-        for prod in products if prod in prod_to_app_name_values
-    }
-
-    products_to_logs = create_key_to_list(products)
-    # Certain events are not being currently being filtered because
-    # filtering for those events here would be duplicative of the Rego code
-    try:
-        # the value we want is nested several layers deep
-        # checks under the APPLICATION_NAME key for the correct app_name value
-        dup_events = (
-            'CHANGE_APPLICATION_SETTING',
-            'CREATE_APPLICATION_SETTING',
-            'DELETE_APPLICATION_SETTING'
+        products_to_logs = create_key_to_list(products)
+        # Certain events are not being currently being filtered because
+        # filtering for those events here would be duplicative of the Rego code
+        try:
+            # the value we want is nested several layers deep
+            # checks under the APPLICATION_NAME key for the correct app_name value
+            dup_events = (
+                'CHANGE_APPLICATION_SETTING',
+                'CREATE_APPLICATION_SETTING',
+                'DELETE_APPLICATION_SETTING'
+                )
+            if event in dup_events:
+                app_name = 'APPLICATION_NAME'
+                for report in response:
+                    for events in report['events']:
+                        parameters = events.get('parameters', [])
+                        for parameter in parameters:
+                            if parameter.get('name') == app_name:
+                                param_val = parameter.get('value')
+                                for prod, app_values in subset_prod_to_app_name.items():
+                                    if param_val in app_values:
+                                        products_to_logs[prod].append(report)
+            else: # no filtering append entire response to relevant product
+                for prod in products:
+                    products_to_logs[prod].extend(response)
+        except Exception as exc:
+            warnings.warn(
+                f"An exception was thrown while getting the logs; outputs will be incorrect: {exc}",
+                RuntimeWarning
             )
-        if event in dup_events:
-            app_name = 'APPLICATION_NAME'
-            for report in response:
-                for events in report['events']:
-                    parameters = events.get('parameters', [])
-                    for parameter in parameters:
-                        if parameter.get('name') == app_name:
-                            param_val = parameter.get('value')
-                            for prod, app_values in subset_prod_to_app_name.items():
-                                if param_val in app_values:
-                                    products_to_logs[prod].append(report)
-        else: # no filtering append entire response to relevant product
-            for prod in products:
-                products_to_logs[prod].extend(response)
-    except Exception as exc:
-        warnings.warn(
-            f"An exception was thrown while getting the logs; outputs will be incorrect: {exc}",
-            RuntimeWarning
-        )
-    return products_to_logs
-
-def get_group_settings(services) -> dict:
-    '''
-    Gets all of the group info using the directory API and group settings API
-
-    :param services: a service instance
-    '''
-
-    try:
-        # set up the services
+        return products_to_logs
+
+    def get_group_settings(self, services) -> dict:
+        '''
+        Gets all of the group info using the directory API and group settings API
+
+        :param services: a service instance
+        '''
+
         group_service = services['groups']
         domain_service = services['directory']
-        # gather all of the domains within a suite to get groups
-        response = domain_service.domains().list(customer="my_customer").execute()
-        domains = {d['domainName'] for d in response['domains'] if d['verified']}
+        try:
+            # gather all of the domains within a suite to get groups
+            response = domain_service.domains().list(customer="my_customer").execute()
+            domains = {d['domainName'] for d in response['domains'] if d['verified']}
+            self.successful_calls.add("directory/v1/domains/list")
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting group settings; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/domains/list")
+            return {'group_settings': []}
+
+        try:
+            # get the group settings for each groups
+            group_settings = []
+            for domain in domains:
+                response = domain_service.groups().list(domain=domain).execute()
+                for group in response.get('groups'):
+                    email = group.get('email')
+                    group_settings.append(group_service.groups().get(groupUniqueId=email).execute())
+            self.successful_calls.add("groups-settings/v1/groups/get")
+            return {'group_settings': group_settings}
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting group settings; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("groups-settings/v1/groups/get")
+            return {'group_settings': []}
+
+    def call_gws_providers(self, products: list, services, quiet) -> dict:
+        '''
+        Calls the relevant GWS APIs to get the data we need for the baselines.
+        Data such as the admin audit log, super admin users etc.
+
+        :param products: list of product names to check
+        :param services: a dict of service objects.
+        :param quiet: suppress tqdm output
+        service is a Google reports API object, created from successfully authenticating in auth.py
+        '''
+        # create a inverse dictionary containing a mapping of event => list of products
+        events_to_products = create_subset_inverted_dict(EVENTS, products)
+        events_to_products_bar = tqdm(events_to_products.items(), leave=False, disable=quiet)
+
+        # main aggregator dict
+        product_to_logs = create_key_to_list(products)
+        product_to_items = {}
+        ou_ids = set()
+        ou_ids.add("") # certain settings have no OU
+        try:
+            # Add top level organization unit name
+            ou_ids.add(self.get_toplevel_ou(services['directory']))
+            # get all organizational unit data
+            product_to_items['organizational_units'] = self.get_ous(services['directory'])
+            for orgunit in product_to_items['organizational_units']['organizationUnits']:
+                ou_ids.add(orgunit['name'])
+            # add just organizational unit names to a field]
+            product_to_items['organizational_unit_names'] = list(ou_ids)
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting tenant data: {exc}",
+                RuntimeWarning
+            )
 
-        # get the group settings for each groups
-        group_settings = []
-        for domain in domains:
-            response = domain_service.groups().list(domain=domain).execute()
-            for group in response.get('groups'):
-                email = group.get('email')
-                group_settings.append(group_service.groups().get(groupUniqueId=email).execute())
-        return {'group_settings': group_settings}
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting group settings; outputs will be incorrect: {exc}",
-            RuntimeWarning
-        )
-        return {'group_settings': []}
-
-def call_gws_providers(products: list, services, quiet) -> dict:
-    '''
-    Calls the relevant GWS APIs to get the data we need for the baselines.
-    Data such as the admin audit log, super admin users etc.
-
-    :param products: list of product names to check
-    :param services: a dict of service objects.
-    :param quiet: suppress tqdm output
-    service is a Google reports API object, created from successfully authenticating in auth.py
-    '''
-    # create a inverse dictionary containing a mapping of event => list of products
-    events_to_products = create_subset_inverted_dict(EVENTS, products)
-    events_to_products_bar = tqdm(events_to_products.items(), leave=False, disable=quiet)
-
-    # main aggregator dict
-    product_to_logs = create_key_to_list(products)
-    product_to_items = {}
-    ou_ids = set()
-    ou_ids.add("") # certain settings have no OU
-    try:
-        # Add top level organization unit name
-        ou_ids.add(get_toplevel_ou(services['directory']))
-        # get all organizational unit data
-        product_to_items['organizational_units'] = get_ous(services['directory'])
-        for orgunit in product_to_items['organizational_units']['organizationUnits']:
-            ou_ids.add(orgunit['name'])
-        # add just organizational unit names to a field]
-        product_to_items['organizational_unit_names'] = list(ou_ids)
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting tenant data: {exc}",
-            RuntimeWarning
-        )
-
-    # call the api once per event type
-    try:
-        for event, product_list in events_to_products_bar:
-            products = ', '.join(product_list)
-            bar_descr = f"Running Provider: Exporting {event} events for {products}..."
-            events_to_products_bar.set_description(bar_descr)
-
-            # gets the GWS admin audit logs and merges them into product_to_logs
-            # aggregator dict
-            product_to_logs = merge_dicts(
-                product_to_logs,
-                get_gws_logs(
-                    products=product_list,
-                    service=services['reports'],
-                    event=event
+        # call the api once per event type
+        try:
+            for event, product_list in events_to_products_bar:
+                products = ', '.join(product_list)
+                bar_descr = f"Running Provider: Exporting {event} events for {products}..."
+                events_to_products_bar.set_description(bar_descr)
+
+                # gets the GWS admin audit logs and merges them into product_to_logs
+                # aggregator dict
+                product_to_logs = merge_dicts(
+                    product_to_logs,
+                    self.get_gws_logs(
+                        products=product_list,
+                        service=services['reports'],
+                        event=event
+                    )
                 )
+                self.successful_calls.add("reports/v1/activity/list")
+        except Exception as exc:
+            warnings.warn(
+                f"Provider Exception thrown while getting the logs; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("reports/v1/activity/list")
+
+        # repacks the main aggregator into the original form
+        # that the api returns the data in; under an 'items' key.
+        # Then we put this key under a {product}_log key for the Rego code
+        try:
+            for product, logs in product_to_logs.items():
+                key_name = f"{product}_logs"
+                product_to_items[key_name] = {'items': logs}
+
+            # get tenant metadata for report front page header
+            product_to_items['tenant_info'] = self.get_tenant_info(services['directory'])
+
+            if 'gmail' in product_to_logs: # add dns info if gmail is being run
+                product_to_items.update(self.get_dnsinfo(services['directory']))
+
+            if 'commoncontrols' in product_to_logs: # add list of super admins if CC is being run
+                product_to_items.update(self.get_super_admins(services['directory']))
+
+            if 'groups' in product_to_logs:
+                product_to_items.update(self.get_group_settings(services=services))
+
+        except Exception as exc:
+            warnings.warn(
+                f"Uncaught Exception thrown while getting other data: {exc}",
+                RuntimeWarning
             )
-    except Exception as exc:
-        warnings.warn(
-            f"Provider Exception thrown while getting the logs; outputs will be incorrect: {exc}",
-            RuntimeWarning
-        )
-
-    # repacks the main aggregator into the original form
-    # that the api returns the data in; under an 'items' key.
-    # Then we put this key under a {product}_log key for the Rego code
-    try:
-        for product, logs in product_to_logs.items():
-            key_name = f"{product}_logs"
-            product_to_items[key_name] = {'items': logs}
-
-        # get tenant metadata for report front page header
-        product_to_items['tenant_info'] = get_tenant_info(services['directory'])
-
-        if 'gmail' in product_to_logs: # add dns info if gmail is being run
-            product_to_items.update(get_dnsinfo(services['directory']))
-
-        if 'commoncontrols' in product_to_logs: # add list of super admins if CC is being run
-            product_to_items.update(get_super_admins(services['directory']))
-
-        if 'groups' in product_to_logs:
-            product_to_items.update(get_group_settings(services=services))
-
-    except Exception as exc:
-        warnings.warn(
-            f"Uncaught Exception thrown while getting other data: {exc}",
-            RuntimeWarning
-        )
-    return product_to_items
+        return product_to_items
diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index 2b329339..e3d3c11b 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -9,6 +9,28 @@
 import pandas as pd
 from scubagoggles.utils import rel_abs_path
 
+def get_reference_a_tag(api_call : str) -> str:
+    '''
+    Craft the link to the documentation page for each API call.
+    
+    :param api_call: a string representing the API call, such as "directory/v1/users/list".
+    '''
+    api = api_call.split('/')[0]
+    call = '/'.join(api_call.split('/')[1:])
+    # All APIs except for the groups-settings api have "rest/" after "reference/"
+    api_type = "" if api == "groups-settings" else "rest/"
+    return f'<a href="https://developers.google.com/admin-sdk/{api}/reference/{api_type}{call}"> \
+        {api_call}</a>'
+
+API_LINKS = {api_call: get_reference_a_tag(api_call) for api_call in [
+    "directory/v1/users/list",
+    "directory/v1/orgunits/list",
+    "directory/v1/domains/list",
+    "directory/v1/groups/list",
+    "reports/v1/activities/list",
+    "group-settings/v1/groups/get"
+]}
+
 def get_test_result(requirement_met : bool, criticality : str, no_such_events : bool) -> str:
     '''
     Checks the Rego to see if the baseline passed or failed and indicates the criticality
@@ -141,7 +163,8 @@ def build_report_html(fragments : list, product : str,
     return html
 
 def rego_json_to_html(test_results_data : str, product : list, out_path : str,
-tenant_domain : str, main_report_name : str, prod_to_fullname: dict, product_policies) -> None:
+tenant_domain : str, main_report_name : str, prod_to_fullname: dict, product_policies,
+successful_calls : set, unsuccessful_calls : set) -> None:
     '''
     Transforms the Rego JSON output into HTML
 
@@ -152,6 +175,8 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str,
     :param main_report_name: report_name: Name of the main report HTML file.
     :param prod_to_fullname: dict containing mapping of the product full names
     :param product_policies: dict containing policies read from the baseline markdown
+    :param successful_calls: set with the set of successful calls
+    :param unsuccessful_calls: set with the set of unsuccessful calls
     '''
 
     product_capitalized = product.capitalize()
@@ -164,7 +189,8 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str,
         "Warning": 0,
         "Fail": 0,
         "N/A": 0,
-        "No events found": 0
+        "No events found": 0,
+        "Error": 0
     }
 
     for baseline_group in product_policies:
@@ -172,48 +198,83 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str,
         for control in baseline_group['Controls']:
             tests = [test for test in test_results_data if test['PolicyId'] == control['Id']]
             for test in tests:
-                result = get_test_result(test['RequirementMet'], test['Criticality'],
-                test['NoSuchEvent'])
-                report_stats[result] = report_stats[result] + 1
-                details = test['ReportDetails']
-
-                if result == "No events found":
-                    warning_icon = "<object data='./images/triangle-exclamation-solid.svg'\
-                        width='15'\
-                        height='15'>\
-                        </object>"
-                    details = warning_icon + " " + test['ReportDetails']
-
-                # As rules doesn't have it's own baseline, Rules and Common Controls
-                # need to be handled specially
-                if product_capitalized == "Rules":
-                    if 'Not-Implemented' in test['Criticality']:
-                        # The easiest way to identify the GWS.COMMONCONTROLS.14.1v1
+                # The following if/else makes the error handling backwards compatible until
+                # all Regos are updated.
+                if 'Prerequisites' not in test:
+                    prereqs = set()
+                else:
+                    prereqs = set(test['Prerequisites'])
+                # A call is failed if it is either missing from the successful_calls set
+                # or present in the unsuccessful_calls
+                failed_calls = set().union(
+                    prereqs.difference(successful_calls),
+                    prereqs.intersection(unsuccessful_calls)
+                )
+                if len(failed_calls) > 0:
+                    result = "Error"
+                    report_stats["Error"] += 1
+                    failed_api_links = [API_LINKS[api] for api in failed_calls if api in API_LINKS]
+                    failed_functions = [call for call in failed_calls if call not in API_LINKS]
+                    failed_details = ""
+                    if len(failed_api_links) > 0:
+                        links = ', '.join(failed_api_links)
+                        failed_details += f"This test depends on the following API call(s) " \
+                            f"which did not execute successfully: {links}. " \
+                            "See terminal output for more details. "
+                    if len(failed_functions) > 0:
+                        failed_details += f"This test depends on the following function(s) " \
+                            f"which did not execute successfully: {', '.join(failed_functions)}." \
+                            "See terminal output for more details."
+                    table_data.append({
+                        'Control ID': control['Id'],
+                        'Requirement': control['Value'],
+                        'Result': "Error",
+                        'Criticality': test['Criticality'],
+                        'Details': failed_details
+                    })
+                else:
+                    result = get_test_result(test['RequirementMet'], test['Criticality'],
+                    test['NoSuchEvent'])
+                    report_stats[result] = report_stats[result] + 1
+                    details = test['ReportDetails']
+
+                    if result == "No events found":
+                        warning_icon = "<object data='./images/triangle-exclamation-solid.svg'\
+                            width='15'\
+                            height='15'>\
+                            </object>"
+                        details = warning_icon + " " + test['ReportDetails']
+
+                    # As rules doesn't have it's own baseline, Rules and Common Controls
+                    # need to be handled specially
+                    if product_capitalized == "Rules":
+                        if 'Not-Implemented' in test['Criticality']:
+                            # The easiest way to identify the GWS.COMMONCONTROLS.14.1v1
+                            # results that belong to the Common Controls report is they're
+                            # marked as Not-Implemented. This if excludes them from the
+                            # rules report.
+                            continue
+                        table_data.append({
+                            'Control ID': control['Id'],
+                            'Rule Name': test['Requirement'],
+                            'Result': result,
+                            'Criticality': test['Criticality'],
+                            'Rule Description': test['ReportDetails']})
+                    elif product_capitalized == "Commoncontrols" \
+                        and baseline_group['GroupName'] == 'System-defined Rules' \
+                        and 'Not-Implemented' not in test['Criticality']:
+                        # The easiest way to identify the System-defined Rules
                         # results that belong to the Common Controls report is they're
-                        # marked as Not-Implemented. This if excludes them from the
-                        # rules report.
+                        # marked as Not-Implemented. This if excludes the full results
+                        # from the Common Controls report.
                         continue
-                    table_data.append({
+                    else:
+                        table_data.append({
                         'Control ID': control['Id'],
-                        'Rule Name': test['Requirement'],
+                        'Requirement': control['Value'],
                         'Result': result,
                         'Criticality': test['Criticality'],
-                        'Rule Description': test['ReportDetails']})
-                elif product_capitalized == "Commoncontrols" \
-                    and baseline_group['GroupName'] == 'System-defined Rules' \
-                    and 'Not-Implemented' not in test['Criticality']:
-                    # The easiest way to identify the System-defined Rules
-                    # results that belong to the Common Controls report is they're
-                    # marked as Not-Implemented. This if excludes the full results
-                    # from the Common Controls report.
-                    continue
-                else:
-                    table_data.append({
-                    'Control ID': control['Id'],
-                    'Requirement': control['Value'],
-                    'Result': result,
-                    'Criticality': test['Criticality'],
-                    'Details': details})
+                        'Details': details})
         fragments.append(f"<h2>{product_upper}-{baseline_group['GroupNumber']} \
         {baseline_group['GroupName']}</h2>")
         fragments.append(create_html_table(table_data))

From 0e1b3010fa13fe986ba9cadd853f145ab0fbfb6a Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Wed, 24 Jan 2024 22:16:06 -0800
Subject: [PATCH 10/35] Fix bad merge

---
 scubagoggles/provider.py          | 21 +--------------------
 scubagoggles/reporter/reporter.py |  4 ----
 2 files changed, 1 insertion(+), 24 deletions(-)

diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index 626b7fe1..d66ea728 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -321,7 +321,6 @@ def get_tenant_info(self, service) -> dict:
         :param service: a directory_v1 service instance
         '''
         try:
-<<<<<<< HEAD
             response = service.domains().list(customer="my_customer").execute()
             self.successful_calls.add("directory/v1/domains/list")
             primary_domain = ""
@@ -332,31 +331,16 @@ def get_tenant_info(self, service) -> dict:
                 'domain': primary_domain,
                 'topLevelOU': self.get_toplevel_ou(service)
             }
-=======
-            response = service.customers().get(customerKey="my_customer").execute()
-            self.successful_calls.add("directory/v1/domains/list")
-            return {'id': response['id'],
-            'domain': response['customerDomain'],
-            'name': response['postalAddress']['organizationName'],
-            'topLevelOU': self.get_toplevel_ou(service)}
->>>>>>> c2770e6c5b19decfe58f4118157ec3bdabbed6a3
         except Exception as exc:
             warnings.warn(
                 f"An exception was thrown trying to get the tenant info: {exc}",
                 RuntimeWarning
             )
             self.unsuccessful_calls.add("directory/v1/domains/list")
-<<<<<<< HEAD
             return {
                 'domain': 'Error Retrieving',
                 'topLevelOU': 'Error Retrieving'
             }
-=======
-            return {'id': 'Error Retrieving',
-            'domain': 'Error Retrieving',
-            'name': 'Error Retrieving',
-            'topLevelOU': 'Error Retrieving'}
->>>>>>> c2770e6c5b19decfe58f4118157ec3bdabbed6a3
 
 
     def get_gws_logs(self, products: list, service, event: str) -> dict:
@@ -446,11 +430,8 @@ def get_group_settings(self, services) -> dict:
         try:
             # gather all of the domains within a suite to get groups
             response = domain_service.domains().list(customer="my_customer").execute()
-<<<<<<< HEAD
             domains = {d['domainName'] for d in response['domains'] if d['verified']}
-=======
-            domains = {d['domainName'] for d in response['domains']}
->>>>>>> c2770e6c5b19decfe58f4118157ec3bdabbed6a3
+
             self.successful_calls.add("directory/v1/domains/list")
         except Exception as exc:
             warnings.warn(
diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index 9a59cf00..e3d3c11b 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -163,11 +163,7 @@ def build_report_html(fragments : list, product : str,
     return html
 
 def rego_json_to_html(test_results_data : str, product : list, out_path : str,
-<<<<<<< HEAD
 tenant_domain : str, main_report_name : str, prod_to_fullname: dict, product_policies,
-=======
-tenant_name : str, main_report_name : str, prod_to_fullname: dict, product_policies,
->>>>>>> c2770e6c5b19decfe58f4118157ec3bdabbed6a3
 successful_calls : set, unsuccessful_calls : set) -> None:
     '''
     Transforms the Rego JSON output into HTML

From 75e8b8a9c3e6ce28187cf0ee47851092d7625f65 Mon Sep 17 00:00:00 2001
From: Alden Hilton <106177711+adhilto@users.noreply.github.com>
Date: Wed, 24 Jan 2024 18:47:47 -0800
Subject: [PATCH 11/35] Simplify Tenant Metadata (#152)

* Simplify tenant metadata section and correct groups bug

* Remove scope that is no longer needed

* Remove unneeded OAUTH scope from readme
---
 README.md                         |  1 -
 scubagoggles/auth.py              |  1 -
 scubagoggles/orchestrator.py      |  4 ++--
 scubagoggles/provider.py          | 24 ++++++++++++++----------
 scubagoggles/reporter/reporter.py | 18 +++++++++---------
 5 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index 49c256df..bc9d91af 100644
--- a/README.md
+++ b/README.md
@@ -142,7 +142,6 @@ chmod +x opa_darwin_amd64 # give the opa executable execute permissions
 The tool uses the following OAUTH API scopes.
 - `https://www.googleapis.com/auth/admin.reports.audit.readonly`
 - `https://www.googleapis.com/auth/admin.directory.domain.readonly`
-- `https://www.googleapis.com/auth/admin.directory.customer.readonly`
 - `https://www.googleapis.com/auth/admin.directory.group.readonly`
 - `https://www.googleapis.com/auth/admin.directory.orgunit.readonly`
 - `https://www.googleapis.com/auth/admin.directory.user.readonly`
diff --git a/scubagoggles/auth.py b/scubagoggles/auth.py
index 892cf3e3..8bea98ec 100644
--- a/scubagoggles/auth.py
+++ b/scubagoggles/auth.py
@@ -14,7 +14,6 @@
 # If modifying these scopes, delete the file token.json.
 SCOPES = ['https://www.googleapis.com/auth/admin.reports.audit.readonly',
     "https://www.googleapis.com/auth/admin.directory.domain.readonly",
-    "https://www.googleapis.com/auth/admin.directory.customer.readonly",
     "https://www.googleapis.com/auth/admin.directory.orgunit.readonly",
     "https://www.googleapis.com/auth/admin.directory.user.readonly",
     "https://www.googleapis.com/auth/admin.directory.group.readonly",
diff --git a/scubagoggles/orchestrator.py b/scubagoggles/orchestrator.py
index b3815b60..def32515 100644
--- a/scubagoggles/orchestrator.py
+++ b/scubagoggles/orchestrator.py
@@ -207,7 +207,7 @@ def run_reporter(args):
     with open(f'{out_folder}/{args.outputproviderfilename}.json',
     mode='r',encoding='UTF-8') as file:
         tenant_info = json.load(file)['tenant_info']
-        tenant_name = tenant_info['name']
+        tenant_domain = tenant_info['domain']
 
 
     # Create the the individual report files
@@ -221,7 +221,7 @@ def run_reporter(args):
             test_results_data,
             product,
             out_folder,
-            tenant_name,
+            tenant_domain,
             main_report_name,
             prod_to_fullname,
             baseline_policies[product]
diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index adb07529..7e765c08 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -303,20 +303,24 @@ def get_tenant_info(service) -> dict:
     :param service: a directory_v1 service instance
     '''
     try:
-        response = service.customers().get(customerKey="my_customer").execute()
-        return {'id': response['id'],
-        'domain': response['customerDomain'],
-        'name': response['postalAddress']['organizationName'],
-        'topLevelOU': get_toplevel_ou(service)}
+        response = service.domains().list(customer="my_customer").execute()
+        primary_domain = ""
+        for domain in response['domains']:
+            if domain['isPrimary']:
+                primary_domain = domain['domainName']
+        return {
+            'domain': primary_domain,
+            'topLevelOU': get_toplevel_ou(service)
+        }
     except Exception as exc:
         warnings.warn(
             f"An exception was thrown trying to get the tenant info: {exc}",
             RuntimeWarning
         )
-        return {'id': 'Error Retrieving',
-        'domain': 'Error Retrieving',
-        'name': 'Error Retrieving',
-        'topLevelOU': 'Error Retrieving'}
+        return {
+            'domain': 'Error Retrieving',
+            'topLevelOU': 'Error Retrieving'
+        }
 
 
 def get_gws_logs(products: list, service, event: str) -> dict:
@@ -407,7 +411,7 @@ def get_group_settings(services) -> dict:
         domain_service = services['directory']
         # gather all of the domains within a suite to get groups
         response = domain_service.domains().list(customer="my_customer").execute()
-        domains = {d['domainName'] for d in response['domains']}
+        domains = {d['domainName'] for d in response['domains'] if d['verified']}
 
         # get the group settings for each groups
         group_settings = []
diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index 564c0f75..276727e8 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -77,20 +77,20 @@ def build_front_page_html(fragments : list, tenant_info : dict) -> str:
     meta_data = f"\
         <table style = \"text-align:center;\"> \
             <colgroup><col/><col/><col/><col/></colgroup> \
-            <tr><th>Customer Name</th><th>Customer Domain</th><th>Customer ID</th><th>Report Date</th></tr> \
-            <tr><td>{tenant_info['name']}</td><td>{tenant_info['domain']}</td><td>{tenant_info['id']}</td><td>{report_date}</td></tr> \
+            <tr><th>Customer Domain</th><th>Report Date</th></tr> \
+            <tr><td>{tenant_info['domain']}</td><td>{report_date}</td></tr> \
         </table>"
     html = html.replace('{{TENANT_DETAILS}}', meta_data)
     return html
 
 def build_report_html(fragments : list, product : str,
-tenant_name : str, main_report_name: str) -> str:
+tenant_domain : str, main_report_name: str) -> str:
     '''
     Adds data into HTML Template and formats the page accordingly
 
     :param fragments: list object containing each baseline
     :param product: str object containing name of Google Product being evaluated
-    :param tenant_name: the name of the tenant.
+    :param tenant_domain: the primary domain of the tenant.
     :param main_report_name: Name of the main report HTML file.
     '''
     reporter_path = str(rel_abs_path(__file__,"./"))
@@ -132,8 +132,8 @@ def build_report_html(fragments : list, product : str,
     meta_data = f"\
         <table style = \"text-align:center;\"> \
             <colgroup><col/><col/><col/></colgroup> \
-            <tr><th>Customer Name </th><th>Report Date</th><th>Baseline Version</th><th>Tool Version</th></tr> \
-            <tr><td>{tenant_name}</td><td>{report_date}</td><td>{baseline_version}</td><td>{tool_version}</td></tr> \
+            <tr><th>Customer Domain </th><th>Report Date</th><th>Baseline Version</th><th>Tool Version</th></tr> \
+            <tr><td>{tenant_domain}</td><td>{report_date}</td><td>{baseline_version}</td><td>{tool_version}</td></tr> \
         </table>"
 
     html = html.replace('{{METADATA}}', meta_data)
@@ -144,14 +144,14 @@ def build_report_html(fragments : list, product : str,
     return html
 
 def rego_json_to_html(test_results_data : str, product : list, out_path : str,
-tenant_name : str, main_report_name : str, prod_to_fullname: dict, product_policies) -> None:
+tenant_domain : str, main_report_name : str, prod_to_fullname: dict, product_policies) -> None:
     '''
     Transforms the Rego JSON output into HTML
 
     :param test_results_data: json object with results of Rego test
     :param product: list of products being tested
     :param out_path: output path where HTML should be saved
-    :param tenant_name: The name of the GWS org
+    :param tenant_domain: The primary domain of the GWS org
     :param main_report_name: report_name: Name of the main report HTML file.
     :param prod_to_fullname: dict containing mapping of the product full names
     :param product_policies: dict containing policies read from the baseline markdown
@@ -236,7 +236,7 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str,
         fragments.append(f"<h2>{product_upper}-{baseline_group['GroupNumber']} \
         {baseline_group['GroupName']}</h2>")
         fragments.append(create_html_table(table_data))
-    html = build_report_html(fragments, prod_to_fullname[product], tenant_name, main_report_name)
+    html = build_report_html(fragments, prod_to_fullname[product], tenant_domain, main_report_name)
     with open(f"{out_path}/IndividualReports/{ind_report_name}",
     mode='w', encoding='UTF-8') as file:
         file.write(html)

From 7228badef231707aac05ac29c682e5ae4add9c7d Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Wed, 24 Jan 2024 17:50:59 -0800
Subject: [PATCH 12/35] Merge latest changes from #158, Ensure Each Control Is
 Included in the Reports

---
 rego/Commoncontrols.rego          |   1 +
 scubagoggles/orchestrator.py      |  20 +-
 scubagoggles/provider.py          | 852 ++++++++++++++++--------------
 scubagoggles/reporter/reporter.py | 137 +++--
 4 files changed, 557 insertions(+), 453 deletions(-)

diff --git a/rego/Commoncontrols.rego b/rego/Commoncontrols.rego
index db577ac7..df2d1278 100644
--- a/rego/Commoncontrols.rego
+++ b/rego/Commoncontrols.rego
@@ -713,6 +713,7 @@ tests contains {
 #--
 tests contains {
     "PolicyId": "GWS.COMMONCONTROLS.7.1v0.1",
+    "Prerequisites": ["directory/v1/users/list"],
     "Criticality": "Shall",
     "ReportDetails": concat("", [
         concat("", ["The following super admins are configured: ", concat(", ", SuperAdmins)]),
diff --git a/scubagoggles/orchestrator.py b/scubagoggles/orchestrator.py
index def32515..3c468e99 100644
--- a/scubagoggles/orchestrator.py
+++ b/scubagoggles/orchestrator.py
@@ -12,7 +12,7 @@
 from googleapiclient.discovery import build
 
 from scubagoggles.auth import gws_auth
-from scubagoggles.provider import call_gws_providers
+from scubagoggles.provider import Provider
 from scubagoggles.run_rego import opa_eval
 from scubagoggles.reporter import reporter, md_parser
 from scubagoggles.utils import rel_abs_path
@@ -63,8 +63,10 @@ def run_gws_providers(args, services):
     out_folder = args.outputpath
     provider_dict = {}
 
-    provider_dict = call_gws_providers(products, services, args.quiet)
-
+    provider = Provider()
+    provider_dict = provider.call_gws_providers(products, services, args.quiet)
+    provider_dict['successful_calls'] = list(provider.successful_calls)
+    provider_dict['unsuccessful_calls'] = list(provider.unsuccessful_calls)
     settings_json = json.dumps(provider_dict, indent = 4)
     out_path = out_folder + f'/{args.outputproviderfilename}.json'
     with open(out_path, mode="w", encoding='UTF-8') as outfile:
@@ -182,8 +184,14 @@ def run_reporter(args):
     with open(test_results_json, mode='r', encoding='UTF-8') as file:
         test_results_data = json.load(file)
 
-    # baseline_path
+    # Get the successful/unsuccessful commands
+    settings_name = f'{out_folder}/{args.outputproviderfilename}.json'
+    with open(settings_name, mode='r', encoding='UTF-8') as file:
+        settings_data = json.load(file)
+    successful_calls = set(settings_data['successful_calls'])
+    unsuccessful_calls = set(settings_data['unsuccessful_calls'])
 
+    # baseline_path
     subset_prod_to_fullname = {
         key: prod_to_fullname[key]
         for key in args.baselines
@@ -224,7 +232,9 @@ def run_reporter(args):
             tenant_domain,
             main_report_name,
             prod_to_fullname,
-            baseline_policies[product]
+            baseline_policies[product],
+            successful_calls,
+            unsuccessful_calls
         )
 
     # Make the report front page
diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index 7e765c08..1a859dc3 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -86,428 +86,460 @@
 
 DNSClient = RobustDNSClient()
 
-def get_spf_records(domains: list) -> list:
-    '''
-    Gets the SPF records for each domain in domains.
-
-    :param domains: The list of domain names (strings).
-    '''
-    results = []
-    n_low_confidence = 0
-    for domain in domains:
-        result = DNSClient.query(domain)
-        if not result['HighConfidence']:
-            n_low_confidence += 1
-        results.append({
-            "domain": domain,
-            "rdata": result["Answers"],
-            "log": result["LogEntries"]
-        })
-    if n_low_confidence > 0:
-        warnings.warn(f"get_spf_records: for {n_low_confidence} domain(s), \
-the traditional DNS queries returned an empty answer \
-section and the DoH queries failed. Will assume SPF not configured, but \
-can't guarantee that failure isn't due to something like split horizon DNS. \
-See ProviderSettingsExport.json under 'spf_records' for more details.", RuntimeWarning)
-    return results
-
-def get_dkim_records(domains : list) -> list:
-    '''
-    Gets the DKIM records for each domain in domains.
-
-    :param domains: The list of domain names (strings).
-    '''
-    results = []
-    n_low_confidence = 0
-    for domain in domains:
-        qnames = [f"{selector}._domainkey.{domain}" for selector in selectors]
-        log_entries = []
-        for qname in qnames:
+class Provider:
+    def __init__(self):
+        self.successful_calls = set()
+        self.unsuccessful_calls = set()
+
+    def get_spf_records(self, domains: list) -> list:
+        '''
+        Gets the SPF records for each domain in domains.
+
+        :param domains: The list of domain names (strings).
+        '''
+        results = []
+        n_low_confidence = 0
+        for domain in domains:
+            result = DNSClient.query(domain)
+            if not result['HighConfidence']:
+                n_low_confidence += 1
+            results.append({
+                "domain": domain,
+                "rdata": result["Answers"],
+                "log": result["LogEntries"]
+            })
+        if n_low_confidence > 0:
+            warnings.warn(f"get_spf_records: for {n_low_confidence} domain(s), \
+    the traditional DNS queries returned an empty answer \
+    section and the DoH queries failed. Will assume SPF not configured, but \
+    can't guarantee that failure isn't due to something like split horizon DNS. \
+    See ProviderSettingsExport.json under 'spf_records' for more details.", RuntimeWarning)
+        return results
+
+    def get_dkim_records(self, domains : list) -> list:
+        '''
+        Gets the DKIM records for each domain in domains.
+
+        :param domains: The list of domain names (strings).
+        '''
+        results = []
+        n_low_confidence = 0
+        for domain in domains:
+            qnames = [f"{selector}._domainkey.{domain}" for selector in selectors]
+            log_entries = []
+            for qname in qnames:
+                result = DNSClient.query(qname)
+                log_entries.extend(result['LogEntries'])
+                if len(result['Answers']) == 0:
+                    # The DKIM record does not exist with this selector, we need to try again with
+                    # a different one
+                    continue
+                # Otherwise, the DKIM record exists with this selector, no need to try the rest
+                break
+
+            if not result['HighConfidence']:
+                n_low_confidence += 1
+            results.append({
+                "domain": domain,
+                "rdata": result["Answers"],
+                "log": log_entries
+            })
+
+        if n_low_confidence > 0:
+            warnings.warn(f"get_dkim_records: for {n_low_confidence} domain(s), \
+    the traditional DNS queries returned an empty answer \
+    section and the DoH queries failed. Will assume DKIM not configured, but \
+    can't guarantee that failure isn't due to something like split horizon DNS. \
+    See ProviderSettingsExport.json under 'dkim_records' for more details.", RuntimeWarning)
+        return results
+
+    def get_dmarc_records(self, domains : list) -> list:
+        '''
+        Gets the DMARC records for each domain in domains.
+
+        :param domains: The list of domain names (strings).
+        '''
+        results = []
+        n_low_confidence = 0
+        for domain in domains:
+            log_entries = []
+            qname = f"_dmarc.{domain}"
             result = DNSClient.query(qname)
             log_entries.extend(result['LogEntries'])
-            if len(result['Answers']) == 0:
-                # The DKIM record does not exist with this selector, we need to try again with
-                # a different one
-                continue
-            # Otherwise, the DKIM record exists with this selector, no need to try the rest
-            break
-
-        if not result['HighConfidence']:
-            n_low_confidence += 1
-        results.append({
-            "domain": domain,
-            "rdata": result["Answers"],
-            "log": log_entries
-        })
-
-    if n_low_confidence > 0:
-        warnings.warn(f"get_dkim_records: for {n_low_confidence} domain(s), \
-the traditional DNS queries returned an empty answer \
-section and the DoH queries failed. Will assume DKIM not configured, but \
-can't guarantee that failure isn't due to something like split horizon DNS. \
-See ProviderSettingsExport.json under 'dkim_records' for more details.", RuntimeWarning)
-    return results
-
-def get_dmarc_records(domains : list) -> list:
-    '''
-    Gets the DMARC records for each domain in domains.
-
-    :param domains: The list of domain names (strings).
-    '''
-    results = []
-    n_low_confidence = 0
-    for domain in domains:
-        log_entries = []
-        qname = f"_dmarc.{domain}"
-        result = DNSClient.query(qname)
-        log_entries.extend(result['LogEntries'])
-        if len(result["Answers"]) == 0:
-            # The domain does not exist. If the record is not available at the full domain
-            # level, we need to check at the organizational domain level.
-            labels = domain.split(".")
-            org_domain = f"{labels[-2]}.{labels[-1]}"
-            result = DNSClient.query(f"_dmarc.{org_domain}")
-            log_entries.extend(result['LogEntries'])
-        if not result['HighConfidence']:
-            n_low_confidence += 1
-        results.append({
-            "domain": domain,
-            "rdata": result["Answers"],
-            "log": log_entries
-        })
-    if n_low_confidence > 0:
-        warnings.warn(f"get_dmarc_records: for {n_low_confidence} domain(s), \
-the traditional DNS queries returned an empty answer \
-section and the DoH queries failed. Will assume DMARC not configured, but \
-can't guarantee that failure isn't due to something like split horizon DNS. \
-See ProviderSettingsExport.json under 'dmarc_records' for more details.", RuntimeWarning)
-    return results
-
-def get_dnsinfo(service):
-    '''
-    Gets DNS Information for Gmail baseline
-
-    :param service: a directory_v1 service instance
-    '''
-    output = {"domains": [], "spf_records": [], "dkim_records": [], "dmarc_records": []}
-
-    # Determine the tenant's domains via the API
-    response = service.domains().list(customer="my_customer").execute()
-    domains = {d['domainName'] for d in response['domains']}
-
-    if len(domains) == 0:
-        warnings.warn("No domains found.", RuntimeWarning)
+            if len(result["Answers"]) == 0:
+                # The domain does not exist. If the record is not available at the full domain
+                # level, we need to check at the organizational domain level.
+                labels = domain.split(".")
+                org_domain = f"{labels[-2]}.{labels[-1]}"
+                result = DNSClient.query(f"_dmarc.{org_domain}")
+                log_entries.extend(result['LogEntries'])
+            if not result['HighConfidence']:
+                n_low_confidence += 1
+            results.append({
+                "domain": domain,
+                "rdata": result["Answers"],
+                "log": log_entries
+            })
+        if n_low_confidence > 0:
+            warnings.warn(f"get_dmarc_records: for {n_low_confidence} domain(s), \
+    the traditional DNS queries returned an empty answer \
+    section and the DoH queries failed. Will assume DMARC not configured, but \
+    can't guarantee that failure isn't due to something like split horizon DNS. \
+    See ProviderSettingsExport.json under 'dmarc_records' for more details.", RuntimeWarning)
+        return results
+
+    def get_dnsinfo(self, service):
+        '''
+        Gets DNS Information for Gmail baseline
+
+        :param service: a directory_v1 service instance
+        '''
+        output = {"domains": [], "spf_records": [], "dkim_records": [], "dmarc_records": []}
+
+        # Determine the tenant's domains via the API
+        response = service.domains().list(customer="my_customer").execute()
+        domains = {d['domainName'] for d in response['domains']}
+
+        if len(domains) == 0:
+            warnings.warn("No domains found.", RuntimeWarning)
+            return output
+
+        output["domains"].extend(domains)
+
+        try:
+            output["spf_records"] = get_spf_records(domains)
+            self.successful_calls.add("get_spf_records")
+        except Exception as exc:
+            output["spf_records"] = []
+            warnings.warn(f"An exception was thrown by get_spf_records: {exc}", RuntimeWarning)
+            self.unsuccessful_calls.add("get_spf_records")
+        try:
+            output["dkim_records"] = get_dkim_records(domains)
+            self.successful_calls.add("get_dkim_records")
+        except Exception as exc:
+            output["dkim_records"] = []
+            warnings.warn(f"An exception was thrown by get_dkim_records: {exc}", RuntimeWarning)
+            self.unsuccessful_calls.add("get_dkim_records")
+        try:
+            output["dmarc_records"] = get_dmarc_records(domains)
+            self.successful_calls.add("get_dmarc_records")
+        except Exception as exc:
+            output["dmarc_records"] = []
+            warnings.warn(f"An exception was thrown by get_dmarc_records: {exc}", RuntimeWarning)
+            self.unsuccessful_calls.add("get_dmarc_records")
         return output
 
-    output["domains"].extend(domains)
-
-    try:
-        output["spf_records"] = get_spf_records(domains)
-    except Exception as exc:
-        output["spf_records"] = []
-        warnings.warn(f"An exception was thrown by get_spf_records: {exc}", RuntimeWarning)
-    try:
-        output["dkim_records"] = get_dkim_records(domains)
-    except Exception as exc:
-        output["dkim_records"] = []
-        warnings.warn(f"An exception was thrown by get_dkim_records: {exc}", RuntimeWarning)
-    try:
-        output["dmarc_records"] = get_dmarc_records(domains)
-    except Exception as exc:
-        output["dmarc_records"] = []
-        warnings.warn(f"An exception was thrown by get_dmarc_records: {exc}", RuntimeWarning)
-    return output
-
-def get_super_admins(service) -> dict:
-    '''
-    Gets the org unit/primary email of all super admins, using the directory API
-
-    :param service: a directory_v1 service instance
-    '''
-    try:
-        response = service.users().list(customer="my_customer", query="isAdmin=True").execute()
-        admins = []
-        for user in response['users']:
-            org_unit = user['orgUnitPath']
-            # strip out the leading '/'
-            org_unit = org_unit[1:] if org_unit.startswith('/') else org_unit
-            email = user['primaryEmail']
-            admins.append({'primaryEmail': email, 'orgUnitPath': org_unit})
-        return {'super_admins': admins}
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting super admins; outputs will be incorrect: {exc}",
-            RuntimeWarning
-        )
-        return {'super_admins': []}
-
-def get_ous(service) -> dict:
-    '''
-    Gets the organizational units using the directory API
-
-    :param service: a directory_v1 service instance
-    '''
-
-    try:
-        response = service.orgunits().list(customerId='my_customer').execute()
-        if 'organizationUnits' not in response:
+    def get_super_admins(self, service) -> dict:
+        '''
+        Gets the org unit/primary email of all super admins, using the directory API
+
+        :param service: a directory_v1 service instance
+        '''
+        try:
+            x/0
+            response = service.users().list(customer="my_customer", query="isAdmin=True").execute()
+            admins = []
+            for user in response['users']:
+                org_unit = user['orgUnitPath']
+                # strip out the leading '/'
+                org_unit = org_unit[1:] if org_unit.startswith('/') else org_unit
+                email = user['primaryEmail']
+                admins.append({'primaryEmail': email, 'orgUnitPath': org_unit})
+            self.successful_calls.add("directory/v1/users/list")
+            return {'super_admins': admins}
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting super admins; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/users/list")
+            return {'super_admins': []}
+
+    def get_ous(self, service) -> dict:
+        '''
+        Gets the organizational units using the directory API
+
+        :param service: a directory_v1 service instance
+        '''
+
+        try:
+            response = service.orgunits().list(customerId='my_customer').execute()
+            self.successful_calls.add("directory/v1/orgunits/list")
+            if 'organizationUnits' not in response:
+                return {}
+            return response
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting top level OU: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/orgunits/list")
             return {}
-        return response
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting top level OU: {exc}",
-            RuntimeWarning
-        )
-        return {}
-
-def get_toplevel_ou(service) -> str:
-    '''
-    Gets the tenant name using the directory API
-
-    :param service: a directory_v1 service instance
-    '''
-
-    try:
-        response = service.orgunits().list(customerId='my_customer',
-                                    orgUnitPath='/',
-                                    type='children').execute()
-        # Because we set orgUnitPath to / and type to children, the API call will only
-        # return the second-level OUs, meaning the parentOrgUnitId of any of the OUs returned
-        # will point us to OU of the entire organization
-        if 'organizationUnits' not in response:
-            # No custom OUs have been created. In this case, we can't
-            # determine the name of the top-level OU. See:
-            # https://stackoverflow.com/questions/26936357/google-directory-api-org-name-of-root-org-unit-path
-            # https://stackoverflow.com/questions/60464432/cannot-get-root-orgunit-in-google-directory-api?noredirect=1&lq=1
-            # Fortunately, when there are no custom OUs present, we won't
-            # need to check if a setting change was made at the top-level
-            # OU in the Rego; because no custom OUs have been created, any
-            # changes have to apply to the top-level OU.
+
+    def get_toplevel_ou(self, service) -> str:
+        '''
+        Gets the tenant name using the directory API
+
+        :param service: a directory_v1 service instance
+        '''
+
+        try:
+            response = service.orgunits().list(customerId='my_customer',
+                                        orgUnitPath='/',
+                                        type='children').execute()
+            # Because we set orgUnitPath to / and type to children, the API call will only
+            # return the second-level OUs, meaning the parentOrgUnitId of any of the OUs returned
+            # will point us to OU of the entire organization
+            if 'organizationUnits' not in response:
+                # No custom OUs have been created. In this case, we can't
+                # determine the name of the top-level OU. See:
+                # https://stackoverflow.com/questions/26936357/google-directory-api-org-name-of-root-org-unit-path
+                # https://stackoverflow.com/questions/60464432/cannot-get-root-orgunit-in-google-directory-api?noredirect=1&lq=1
+                # Fortunately, when there are no custom OUs present, we won't
+                # need to check if a setting change was made at the top-level
+                # OU in the Rego; because no custom OUs have been created, any
+                # changes have to apply to the top-level OU.
+                return ""
+            parent_ou = response['organizationUnits'][0]['parentOrgUnitId']
+            response = service.orgunits().get(customerId='my_customer', orgUnitPath=parent_ou).execute()
+            ou_name = response['name']
+            self.successful_calls.add("directory/v1/orgunits/list")
+            return ou_name
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting top level OU: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/orgunits/list")
             return ""
-        parent_ou = response['organizationUnits'][0]['parentOrgUnitId']
-        response = service.orgunits().get(customerId='my_customer', orgUnitPath=parent_ou).execute()
-        ou_name = response['name']
-        return ou_name
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting top level OU: {exc}",
-            RuntimeWarning
-        )
-        return ""
-
-
-def get_tenant_info(service) -> dict:
-    '''
-    Gets the high-level tenant info using the directory API
-
-    :param service: a directory_v1 service instance
-    '''
-    try:
-        response = service.domains().list(customer="my_customer").execute()
-        primary_domain = ""
-        for domain in response['domains']:
-            if domain['isPrimary']:
-                primary_domain = domain['domainName']
-        return {
-            'domain': primary_domain,
-            'topLevelOU': get_toplevel_ou(service)
+
+
+    def get_tenant_info(self, service) -> dict:
+        '''
+        Gets the high-level tenant info using the directory API
+
+        :param service: a directory_v1 service instance
+        '''
+        try:
+            response = service.domains().list(customer="my_customer").execute()
+            self.successful_calls.add("directory/v1/domains/list")
+            primary_domain = ""
+            for domain in response['domains']:
+                if domain['isPrimary']:
+                    primary_domain = domain['domainName']
+            return {
+                'domain': primary_domain,
+                'topLevelOU': self.get_toplevel_ou(service)
+            }
+        except Exception as exc:
+            warnings.warn(
+                f"An exception was thrown trying to get the tenant info: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/domains/list")
+            return {
+                'domain': 'Error Retrieving',
+                'topLevelOU': 'Error Retrieving'
+            }
+
+
+    def get_gws_logs(self, products: list, service, event: str) -> dict:
+        '''
+        Gets the GWS admin audit logs with the specified event name.
+        This function will also some parsing and filtering to ensure that an appropriate
+        log event is matched to the appropriate product.
+        This is to prevent the same log event from being duplicated
+        across products in the resulting provider JSON.
+
+        :param products: a narrowed list of the products being invoked
+        :param service: service is a Google reports API object, created from successfully
+        authenticating in auth.py
+        :param event: the name of the specific event we are querying for.
+        '''
+
+        # Filter responses by org_unit id
+        response = (service.activities().list(userKey='all',
+                applicationName='admin',
+                eventName=event).execute()).get('items', [])
+
+
+        # Used for filtering duplicate events
+        prod_to_app_name_values = {
+            'calendar': ['Calendar'],
+            'chat': ['Google Chat', 'Google Workspace Marketplace'],
+            'commoncontrols': [
+                'Security',
+                'Google Workspace Marketplace',
+                'Blogger',
+                'Google Cloud Platform Sharing Options',
+            ],
+            'drive': ['Drive and Docs'],
+            'gmail': ['Gmail'],
+            'groups': ['Groups for Business'],
+            'meet': ['Google Meet'],
+            'sites': ['Sites'],
+            'classroom': ['Classroom']
         }
-    except Exception as exc:
-        warnings.warn(
-            f"An exception was thrown trying to get the tenant info: {exc}",
-            RuntimeWarning
-        )
-        return {
-            'domain': 'Error Retrieving',
-            'topLevelOU': 'Error Retrieving'
+        # create a subset of just the products we need from the dict above
+        subset_prod_to_app_name = {
+            prod: prod_to_app_name_values[prod]
+            for prod in products if prod in prod_to_app_name_values
         }
 
-
-def get_gws_logs(products: list, service, event: str) -> dict:
-    '''
-    Gets the GWS admin audit logs with the specified event name.
-    This function will also some parsing and filtering to ensure that an appropriate
-    log event is matched to the appropriate product.
-    This is to prevent the same log event from being duplicated
-    across products in the resulting provider JSON.
-
-    :param products: a narrowed list of the products being invoked
-    :param service: service is a Google reports API object, created from successfully
-    authenticating in auth.py
-    :param event: the name of the specific event we are querying for.
-    '''
-
-    # Filter responses by org_unit id
-    response = (service.activities().list(userKey='all',
-            applicationName='admin',
-            eventName=event).execute()).get('items', [])
-
-
-    # Used for filtering duplicate events
-    prod_to_app_name_values = {
-        'calendar': ['Calendar'],
-        'chat': ['Google Chat', 'Google Workspace Marketplace'],
-        'commoncontrols': [
-            'Security',
-            'Google Workspace Marketplace',
-            'Blogger',
-            'Google Cloud Platform Sharing Options',
-        ],
-        'drive': ['Drive and Docs'],
-        'gmail': ['Gmail'],
-        'groups': ['Groups for Business'],
-        'meet': ['Google Meet'],
-        'sites': ['Sites'],
-        'classroom': ['Classroom']
-    }
-    # create a subset of just the products we need from the dict above
-    subset_prod_to_app_name = {
-        prod: prod_to_app_name_values[prod]
-        for prod in products if prod in prod_to_app_name_values
-    }
-
-    products_to_logs = create_key_to_list(products)
-    # Certain events are not being currently being filtered because
-    # filtering for those events here would be duplicative of the Rego code
-    try:
-        # the value we want is nested several layers deep
-        # checks under the APPLICATION_NAME key for the correct app_name value
-        dup_events = (
-            'CHANGE_APPLICATION_SETTING',
-            'CREATE_APPLICATION_SETTING',
-            'DELETE_APPLICATION_SETTING'
+        products_to_logs = create_key_to_list(products)
+        # Certain events are not being currently being filtered because
+        # filtering for those events here would be duplicative of the Rego code
+        try:
+            # the value we want is nested several layers deep
+            # checks under the APPLICATION_NAME key for the correct app_name value
+            dup_events = (
+                'CHANGE_APPLICATION_SETTING',
+                'CREATE_APPLICATION_SETTING',
+                'DELETE_APPLICATION_SETTING'
+                )
+            if event in dup_events:
+                app_name = 'APPLICATION_NAME'
+                for report in response:
+                    for events in report['events']:
+                        parameters = events.get('parameters', [])
+                        for parameter in parameters:
+                            if parameter.get('name') == app_name:
+                                param_val = parameter.get('value')
+                                for prod, app_values in subset_prod_to_app_name.items():
+                                    if param_val in app_values:
+                                        products_to_logs[prod].append(report)
+            else: # no filtering append entire response to relevant product
+                for prod in products:
+                    products_to_logs[prod].extend(response)
+        except Exception as exc:
+            warnings.warn(
+                f"An exception was thrown while getting the logs; outputs will be incorrect: {exc}",
+                RuntimeWarning
             )
-        if event in dup_events:
-            app_name = 'APPLICATION_NAME'
-            for report in response:
-                for events in report['events']:
-                    parameters = events.get('parameters', [])
-                    for parameter in parameters:
-                        if parameter.get('name') == app_name:
-                            param_val = parameter.get('value')
-                            for prod, app_values in subset_prod_to_app_name.items():
-                                if param_val in app_values:
-                                    products_to_logs[prod].append(report)
-        else: # no filtering append entire response to relevant product
-            for prod in products:
-                products_to_logs[prod].extend(response)
-    except Exception as exc:
-        warnings.warn(
-            f"An exception was thrown while getting the logs; outputs will be incorrect: {exc}",
-            RuntimeWarning
-        )
-    return products_to_logs
-
-def get_group_settings(services) -> dict:
-    '''
-    Gets all of the group info using the directory API and group settings API
-
-    :param services: a service instance
-    '''
-
-    try:
-        # set up the services
+        return products_to_logs
+
+    def get_group_settings(self, services) -> dict:
+        '''
+        Gets all of the group info using the directory API and group settings API
+
+        :param services: a service instance
+        '''
+
         group_service = services['groups']
         domain_service = services['directory']
-        # gather all of the domains within a suite to get groups
-        response = domain_service.domains().list(customer="my_customer").execute()
-        domains = {d['domainName'] for d in response['domains'] if d['verified']}
+        try:
+            # gather all of the domains within a suite to get groups
+            response = domain_service.domains().list(customer="my_customer").execute()
+            domains = {d['domainName'] for d in response['domains'] if d['verified']}
+            self.successful_calls.add("directory/v1/domains/list")
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting group settings; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("directory/v1/domains/list")
+            return {'group_settings': []}
+
+        try:
+            # get the group settings for each groups
+            group_settings = []
+            for domain in domains:
+                response = domain_service.groups().list(domain=domain).execute()
+                for group in response.get('groups'):
+                    email = group.get('email')
+                    group_settings.append(group_service.groups().get(groupUniqueId=email).execute())
+            self.successful_calls.add("groups-settings/v1/groups/get")
+            return {'group_settings': group_settings}
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting group settings; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("groups-settings/v1/groups/get")
+            return {'group_settings': []}
+
+    def call_gws_providers(self, products: list, services, quiet) -> dict:
+        '''
+        Calls the relevant GWS APIs to get the data we need for the baselines.
+        Data such as the admin audit log, super admin users etc.
+
+        :param products: list of product names to check
+        :param services: a dict of service objects.
+        :param quiet: suppress tqdm output
+        service is a Google reports API object, created from successfully authenticating in auth.py
+        '''
+        # create a inverse dictionary containing a mapping of event => list of products
+        events_to_products = create_subset_inverted_dict(EVENTS, products)
+        events_to_products_bar = tqdm(events_to_products.items(), leave=False, disable=quiet)
+
+        # main aggregator dict
+        product_to_logs = create_key_to_list(products)
+        product_to_items = {}
+        ou_ids = set()
+        ou_ids.add("") # certain settings have no OU
+        try:
+            # Add top level organization unit name
+            ou_ids.add(self.get_toplevel_ou(services['directory']))
+            # get all organizational unit data
+            product_to_items['organizational_units'] = self.get_ous(services['directory'])
+            for orgunit in product_to_items['organizational_units']['organizationUnits']:
+                ou_ids.add(orgunit['name'])
+            # add just organizational unit names to a field]
+            product_to_items['organizational_unit_names'] = list(ou_ids)
+        except Exception as exc:
+            warnings.warn(
+                f"Exception thrown while getting tenant data: {exc}",
+                RuntimeWarning
+            )
 
-        # get the group settings for each groups
-        group_settings = []
-        for domain in domains:
-            response = domain_service.groups().list(domain=domain).execute()
-            for group in response.get('groups'):
-                email = group.get('email')
-                group_settings.append(group_service.groups().get(groupUniqueId=email).execute())
-        return {'group_settings': group_settings}
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting group settings; outputs will be incorrect: {exc}",
-            RuntimeWarning
-        )
-        return {'group_settings': []}
-
-def call_gws_providers(products: list, services, quiet) -> dict:
-    '''
-    Calls the relevant GWS APIs to get the data we need for the baselines.
-    Data such as the admin audit log, super admin users etc.
-
-    :param products: list of product names to check
-    :param services: a dict of service objects.
-    :param quiet: suppress tqdm output
-    service is a Google reports API object, created from successfully authenticating in auth.py
-    '''
-    # create a inverse dictionary containing a mapping of event => list of products
-    events_to_products = create_subset_inverted_dict(EVENTS, products)
-    events_to_products_bar = tqdm(events_to_products.items(), leave=False, disable=quiet)
-
-    # main aggregator dict
-    product_to_logs = create_key_to_list(products)
-    product_to_items = {}
-    ou_ids = set()
-    ou_ids.add("") # certain settings have no OU
-    try:
-        # Add top level organization unit name
-        ou_ids.add(get_toplevel_ou(services['directory']))
-        # get all organizational unit data
-        product_to_items['organizational_units'] = get_ous(services['directory'])
-        for orgunit in product_to_items['organizational_units']['organizationUnits']:
-            ou_ids.add(orgunit['name'])
-        # add just organizational unit names to a field]
-        product_to_items['organizational_unit_names'] = list(ou_ids)
-    except Exception as exc:
-        warnings.warn(
-            f"Exception thrown while getting tenant data: {exc}",
-            RuntimeWarning
-        )
-
-    # call the api once per event type
-    try:
-        for event, product_list in events_to_products_bar:
-            products = ', '.join(product_list)
-            bar_descr = f"Running Provider: Exporting {event} events for {products}..."
-            events_to_products_bar.set_description(bar_descr)
-
-            # gets the GWS admin audit logs and merges them into product_to_logs
-            # aggregator dict
-            product_to_logs = merge_dicts(
-                product_to_logs,
-                get_gws_logs(
-                    products=product_list,
-                    service=services['reports'],
-                    event=event
+        # call the api once per event type
+        try:
+            for event, product_list in events_to_products_bar:
+                products = ', '.join(product_list)
+                bar_descr = f"Running Provider: Exporting {event} events for {products}..."
+                events_to_products_bar.set_description(bar_descr)
+
+                # gets the GWS admin audit logs and merges them into product_to_logs
+                # aggregator dict
+                product_to_logs = merge_dicts(
+                    product_to_logs,
+                    self.get_gws_logs(
+                        products=product_list,
+                        service=services['reports'],
+                        event=event
+                    )
                 )
+                self.successful_calls.add("reports/v1/activity/list")
+        except Exception as exc:
+            warnings.warn(
+                f"Provider Exception thrown while getting the logs; outputs will be incorrect: {exc}",
+                RuntimeWarning
+            )
+            self.unsuccessful_calls.add("reports/v1/activity/list")
+
+        # repacks the main aggregator into the original form
+        # that the api returns the data in; under an 'items' key.
+        # Then we put this key under a {product}_log key for the Rego code
+        try:
+            for product, logs in product_to_logs.items():
+                key_name = f"{product}_logs"
+                product_to_items[key_name] = {'items': logs}
+
+            # get tenant metadata for report front page header
+            product_to_items['tenant_info'] = self.get_tenant_info(services['directory'])
+
+            if 'gmail' in product_to_logs: # add dns info if gmail is being run
+                product_to_items.update(self.get_dnsinfo(services['directory']))
+
+            if 'commoncontrols' in product_to_logs: # add list of super admins if CC is being run
+                product_to_items.update(self.get_super_admins(services['directory']))
+
+            if 'groups' in product_to_logs:
+                product_to_items.update(self.get_group_settings(services=services))
+
+        except Exception as exc:
+            warnings.warn(
+                f"Uncaught Exception thrown while getting other data: {exc}",
+                RuntimeWarning
             )
-    except Exception as exc:
-        warnings.warn(
-            f"Provider Exception thrown while getting the logs; outputs will be incorrect: {exc}",
-            RuntimeWarning
-        )
-
-    # repacks the main aggregator into the original form
-    # that the api returns the data in; under an 'items' key.
-    # Then we put this key under a {product}_log key for the Rego code
-    try:
-        for product, logs in product_to_logs.items():
-            key_name = f"{product}_logs"
-            product_to_items[key_name] = {'items': logs}
-
-        # get tenant metadata for report front page header
-        product_to_items['tenant_info'] = get_tenant_info(services['directory'])
-
-        if 'gmail' in product_to_logs: # add dns info if gmail is being run
-            product_to_items.update(get_dnsinfo(services['directory']))
-
-        if 'commoncontrols' in product_to_logs: # add list of super admins if CC is being run
-            product_to_items.update(get_super_admins(services['directory']))
-
-        if 'groups' in product_to_logs:
-            product_to_items.update(get_group_settings(services=services))
-
-    except Exception as exc:
-        warnings.warn(
-            f"Uncaught Exception thrown while getting other data: {exc}",
-            RuntimeWarning
-        )
-    return product_to_items
+        return product_to_items
diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index 276727e8..132fa08d 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -12,6 +12,28 @@
 
 SCUBA_GITHUB_URL = "https://github.com/cisagov/scubagoggles"
 
+def get_reference_a_tag(api_call : str) -> str:
+    '''
+    Craft the link to the documentation page for each API call.
+    
+    :param api_call: a string representing the API call, such as "directory/v1/users/list".
+    '''
+    api = api_call.split('/')[0]
+    call = '/'.join(api_call.split('/')[1:])
+    # All APIs except for the groups-settings api have "rest/" after "reference/"
+    api_type = "" if api == "groups-settings" else "rest/"
+    return f'<a href="https://developers.google.com/admin-sdk/{api}/reference/{api_type}{call}"> \
+        {api_call}</a>'
+
+API_LINKS = {api_call: get_reference_a_tag(api_call) for api_call in [
+    "directory/v1/users/list",
+    "directory/v1/orgunits/list",
+    "directory/v1/domains/list",
+    "directory/v1/groups/list",
+    "reports/v1/activities/list",
+    "group-settings/v1/groups/get"
+]}
+
 def get_test_result(requirement_met : bool, criticality : str, no_such_events : bool) -> str:
     '''
     Checks the Rego to see if the baseline passed or failed and indicates the criticality
@@ -144,7 +166,8 @@ def build_report_html(fragments : list, product : str,
     return html
 
 def rego_json_to_html(test_results_data : str, product : list, out_path : str,
-tenant_domain : str, main_report_name : str, prod_to_fullname: dict, product_policies) -> None:
+tenant_domain : str, main_report_name : str, prod_to_fullname: dict, product_policies,
+successful_calls : set, unsuccessful_calls : set) -> None:
     '''
     Transforms the Rego JSON output into HTML
 
@@ -155,6 +178,8 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str,
     :param main_report_name: report_name: Name of the main report HTML file.
     :param prod_to_fullname: dict containing mapping of the product full names
     :param product_policies: dict containing policies read from the baseline markdown
+    :param successful_calls: set with the set of successful calls
+    :param unsuccessful_calls: set with the set of unsuccessful calls
     '''
 
     product_capitalized = product.capitalize()
@@ -190,49 +215,85 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str,
                     RuntimeWarning)
             else:
                 for test in tests:
-                    result = get_test_result(test['RequirementMet'], test['Criticality'],
-                        test['NoSuchEvent'])
-                    report_stats[result] = report_stats[result] + 1
-                    details = test['ReportDetails']
-
-                    if result == "No events found":
-                        warning_icon = "<object data='./images/triangle-exclamation-solid.svg'\
-                            width='15'\
-                            height='15'>\
-                            </object>"
-                        details = warning_icon + " " + test['ReportDetails']
-
-                    # As rules doesn't have its own baseline, Rules and Common Controls
-                    # need to be handled specially
-                    if product_capitalized == "Rules":
-                        if 'Not-Implemented' in test['Criticality']:
-                            # The easiest way to identify the GWS.COMMONCONTROLS.13.1v1
-                            # results that belong to the Common Controls report is they're
-                            # marked as Not-Implemented. This if excludes them from the
-                            # rules report.
-                            continue
-                        table_data.append({
-                            'Control ID': control['Id'],
-                            'Rule Name': test['Requirement'],
-                            'Result': result,
-                            'Criticality': test['Criticality'],
-                            'Rule Description': test['ReportDetails']})
-                    elif product_capitalized == "Commoncontrols" \
-                        and baseline_group['GroupName'] == 'System-defined Rules' \
-                        and 'Not-Implemented' not in test['Criticality']:
-                        # The easiest way to identify the System-defined Rules
-                        # results that belong to the Common Controls report is they're
-                        # marked as Not-Implemented. This if excludes the full results
-                        # from the Common Controls report.
-                        continue
+                    # The following if/else makes the error handling backwards compatible until
+                    # all Regos are updated.
+                    if 'Prerequisites' not in test:
+                        prereqs = set()
                     else:
+                        prereqs = set(test['Prerequisites'])
+                    # A call is failed if it is either missing from the successful_calls set
+                    # or present in the unsuccessful_calls
+                    failed_calls = set().union(
+                        prereqs.difference(successful_calls),
+                        prereqs.intersection(unsuccessful_calls)
+                    )
+                    if len(failed_calls) > 0:
+                        result = "Error"
+                        report_stats["Error"] += 1
+                        failed_api_links = [API_LINKS[api] for api in failed_calls if api in API_LINKS]
+                        failed_functions = [call for call in failed_calls if call not in API_LINKS]
+                        failed_details = ""
+                        if len(failed_api_links) > 0:
+                            links = ', '.join(failed_api_links)
+                            failed_details += f"This test depends on the following API call(s) " \
+                                f"which did not execute successfully: {links}. " \
+                                "See terminal output for more details. "
+                        if len(failed_functions) > 0:
+                            failed_details += f"This test depends on the following function(s) " \
+                                f"which did not execute successfully: {', '.join(failed_functions)}." \
+                                "See terminal output for more details."
                         table_data.append({
                             'Control ID': control['Id'],
                             'Requirement': control['Value'],
-                            'Result': result,
+                            'Result': "Error",
                             'Criticality': test['Criticality'],
-                            'Details': details
+                            'Details': failed_details
                         })
+                    else:
+                        result = get_test_result(test['RequirementMet'], test['Criticality'],
+                        test['NoSuchEvent'])
+
+                        report_stats[result] = report_stats[result] + 1
+                        details = test['ReportDetails']
+
+                        if result == "No events found":
+                            warning_icon = "<object data='./images/triangle-exclamation-solid.svg'\
+                                width='15'\
+                                height='15'>\
+                                </object>"
+                            details = warning_icon + " " + test['ReportDetails']
+
+                        # As rules doesn't have its own baseline, Rules and Common Controls
+                        # need to be handled specially
+                        if product_capitalized == "Rules":
+                            if 'Not-Implemented' in test['Criticality']:
+                                # The easiest way to identify the GWS.COMMONCONTROLS.13.1v1
+                                # results that belong to the Common Controls report is they're
+                                # marked as Not-Implemented. This if excludes them from the
+                                # rules report.
+                                continue
+                            table_data.append({
+                                'Control ID': control['Id'],
+                                'Rule Name': test['Requirement'],
+                                'Result': result,
+                                'Criticality': test['Criticality'],
+                                'Rule Description': test['ReportDetails']})
+                        elif product_capitalized == "Commoncontrols" \
+                            and baseline_group['GroupName'] == 'System-defined Rules' \
+                            and 'Not-Implemented' not in test['Criticality']:
+                            # The easiest way to identify the System-defined Rules
+                            # results that belong to the Common Controls report is they're
+                            # marked as Not-Implemented. This if excludes the full results
+                            # from the Common Controls report.
+                            continue
+                        else:
+                            table_data.append({
+                                'Control ID': control['Id'],
+                                'Requirement': control['Value'],
+                                'Result': result,
+                                'Criticality': test['Criticality'],
+                                'Details': details
+                            })
         fragments.append(f"<h2>{product_upper}-{baseline_group['GroupNumber']} \
         {baseline_group['GroupName']}</h2>")
         fragments.append(create_html_table(table_data))

From e071a83214aec62eedf00fd505d1ff10120a4f42 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Wed, 24 Jan 2024 22:16:06 -0800
Subject: [PATCH 13/35] Fix bad merge

---
 scubagoggles/provider.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index 1a859dc3..d66ea728 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -431,6 +431,7 @@ def get_group_settings(self, services) -> dict:
             # gather all of the domains within a suite to get groups
             response = domain_service.domains().list(customer="my_customer").execute()
             domains = {d['domainName'] for d in response['domains'] if d['verified']}
+
             self.successful_calls.add("directory/v1/domains/list")
         except Exception as exc:
             warnings.warn(

From 4b3573b58d5caa4695477266e512a4b9c53d0237 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 25 Jan 2024 07:58:35 -0800
Subject: [PATCH 14/35] Correct gmail bug, forgot to preface the dns function
 calls with self

---
 scubagoggles/provider.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index d66ea728..8c7a2846 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -207,21 +207,21 @@ def get_dnsinfo(self, service):
         output["domains"].extend(domains)
 
         try:
-            output["spf_records"] = get_spf_records(domains)
+            output["spf_records"] = self.get_spf_records(domains)
             self.successful_calls.add("get_spf_records")
         except Exception as exc:
             output["spf_records"] = []
             warnings.warn(f"An exception was thrown by get_spf_records: {exc}", RuntimeWarning)
             self.unsuccessful_calls.add("get_spf_records")
         try:
-            output["dkim_records"] = get_dkim_records(domains)
+            output["dkim_records"] = self.get_dkim_records(domains)
             self.successful_calls.add("get_dkim_records")
         except Exception as exc:
             output["dkim_records"] = []
             warnings.warn(f"An exception was thrown by get_dkim_records: {exc}", RuntimeWarning)
             self.unsuccessful_calls.add("get_dkim_records")
         try:
-            output["dmarc_records"] = get_dmarc_records(domains)
+            output["dmarc_records"] = self.get_dmarc_records(domains)
             self.successful_calls.add("get_dmarc_records")
         except Exception as exc:
             output["dmarc_records"] = []

From a3dcb3a8994f766b8c04aed29bd908a343b05bf9 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 25 Jan 2024 08:04:37 -0800
Subject: [PATCH 15/35] Refactor long lines

---
 scubagoggles/provider.py          | 14 ++++++++------
 scubagoggles/reporter/reporter.py |  4 +++-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index 8c7a2846..c0bd01c0 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -87,6 +87,10 @@
 DNSClient = RobustDNSClient()
 
 class Provider:
+    '''
+    Class for making the GWS api calls and tracking the results.
+    '''
+
     def __init__(self):
         self.successful_calls = set()
         self.unsuccessful_calls = set()
@@ -236,7 +240,6 @@ def get_super_admins(self, service) -> dict:
         :param service: a directory_v1 service instance
         '''
         try:
-            x/0
             response = service.users().list(customer="my_customer", query="isAdmin=True").execute()
             admins = []
             for user in response['users']:
@@ -301,7 +304,8 @@ def get_toplevel_ou(self, service) -> str:
                 # changes have to apply to the top-level OU.
                 return ""
             parent_ou = response['organizationUnits'][0]['parentOrgUnitId']
-            response = service.orgunits().get(customerId='my_customer', orgUnitPath=parent_ou).execute()
+            response = service.orgunits().get(customerId='my_customer', orgUnitPath=parent_ou)\
+                .execute()
             ou_name = response['name']
             self.successful_calls.add("directory/v1/orgunits/list")
             return ou_name
@@ -512,10 +516,8 @@ def call_gws_providers(self, products: list, services, quiet) -> dict:
                 )
                 self.successful_calls.add("reports/v1/activity/list")
         except Exception as exc:
-            warnings.warn(
-                f"Provider Exception thrown while getting the logs; outputs will be incorrect: {exc}",
-                RuntimeWarning
-            )
+            warnings.warn(f"Provider Exception thrown while getting the logs; "\
+                "outputs will be incorrect: {exc}", RuntimeWarning)
             self.unsuccessful_calls.add("reports/v1/activity/list")
 
         # repacks the main aggregator into the original form
diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index 132fa08d..4ad06375 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -230,7 +230,9 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str,
                     if len(failed_calls) > 0:
                         result = "Error"
                         report_stats["Error"] += 1
-                        failed_api_links = [API_LINKS[api] for api in failed_calls if api in API_LINKS]
+                        failed_api_links = [
+                            API_LINKS[api] for api in failed_calls if api in API_LINKS
+                        ]
                         failed_functions = [call for call in failed_calls if call not in API_LINKS]
                         failed_details = ""
                         if len(failed_api_links) > 0:

From 3a29cbe4aeafc4377ea949404fe3be5109be29d7 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 25 Jan 2024 08:27:35 -0800
Subject: [PATCH 16/35] Broke out error handling code into separate functions

---
 scubagoggles/reporter/reporter.py | 121 ++++++++++++++++++------------
 1 file changed, 72 insertions(+), 49 deletions(-)

diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index 4ad06375..2a42e79a 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -12,28 +12,6 @@
 
 SCUBA_GITHUB_URL = "https://github.com/cisagov/scubagoggles"
 
-def get_reference_a_tag(api_call : str) -> str:
-    '''
-    Craft the link to the documentation page for each API call.
-    
-    :param api_call: a string representing the API call, such as "directory/v1/users/list".
-    '''
-    api = api_call.split('/')[0]
-    call = '/'.join(api_call.split('/')[1:])
-    # All APIs except for the groups-settings api have "rest/" after "reference/"
-    api_type = "" if api == "groups-settings" else "rest/"
-    return f'<a href="https://developers.google.com/admin-sdk/{api}/reference/{api_type}{call}"> \
-        {api_call}</a>'
-
-API_LINKS = {api_call: get_reference_a_tag(api_call) for api_call in [
-    "directory/v1/users/list",
-    "directory/v1/orgunits/list",
-    "directory/v1/domains/list",
-    "directory/v1/groups/list",
-    "reports/v1/activities/list",
-    "group-settings/v1/groups/get"
-]}
-
 def get_test_result(requirement_met : bool, criticality : str, no_such_events : bool) -> str:
     '''
     Checks the Rego to see if the baseline passed or failed and indicates the criticality
@@ -165,6 +143,75 @@ def build_report_html(fragments : list, product : str,
     html = html.replace('{{TABLES}}', collected)
     return html
 
+def get_failed_prereqs(test : dict, successful_calls : set, unsuccessful_calls : set) -> set:
+    '''
+    Given the output of a specific Rego test and the set of successful and unsuccessful
+    calls, determine the set of prerequisites that were not met.
+    :param test: a dictionary representing the output of a Rego test
+    :param successful_calls: a set with the successful provider calls
+    :param unsuccessful_calls: a set with the unsuccessful provider calls
+    '''
+    # The following if/else makes the error handling backwards compatible until
+    # all Regos are updated.
+    if 'Prerequisites' not in test:
+        prereqs = set()
+    else:
+        prereqs = set(test['Prerequisites'])
+
+    # A call is failed if it is either missing from the successful_calls set
+    # or present in the unsuccessful_calls
+    failed_prereqs = set().union(
+        prereqs.difference(successful_calls),
+        prereqs.intersection(unsuccessful_calls)
+    )
+
+    return failed_prereqs
+
+def get_reference_a_tag(api_call : str) -> str:
+    '''
+    Craft the link to the documentation page for a given API call.
+    
+    :param api_call: a string representing the API call, such as "directory/v1/users/list".
+    '''
+    api = api_call.split('/')[0]
+    call = '/'.join(api_call.split('/')[1:])
+    # All APIs except for the groups-settings api have "rest/" after "reference/"
+    api_type = "" if api == "groups-settings" else "rest/"
+    return f'<a href="https://developers.google.com/admin-sdk/{api}/reference/{api_type}{call}"> \
+        {api_call}</a>'
+
+def get_failed_details(failed_prereqs : set) -> str:
+    '''
+    Create the string used for the Details column of the report when one
+    or more of the API calls/functions failed.
+
+    :param failed_prereqs: A set of strings with the API calls/function prerequisites
+        that were not met for a given test.
+    '''
+    api_links = {
+        api_call: get_reference_a_tag(api_call) for api_call in [
+            "directory/v1/users/list",
+            "directory/v1/orgunits/list",
+            "directory/v1/domains/list",
+            "directory/v1/groups/list",
+            "reports/v1/activities/list",
+            "group-settings/v1/groups/get"
+        ]
+    }
+
+    failed_apis = [api_links[api] for api in failed_prereqs if api in api_links]
+    failed_functions = [call for call in failed_prereqs if call not in api_links]
+    failed_details = ""
+    if len(failed_apis) > 0:
+        links = ', '.join(failed_apis)
+        failed_details += f"This test depends on the following API call(s) " \
+            f"which did not execute successfully: {links}. "
+    if len(failed_functions) > 0:
+        failed_details += f"This test depends on the following function(s) " \
+            f"which did not execute successfully: {', '.join(failed_functions)}. "
+    failed_details += "See terminal output for more details."
+    return failed_details
+
 def rego_json_to_html(test_results_data : str, product : list, out_path : str,
 tenant_domain : str, main_report_name : str, prod_to_fullname: dict, product_policies,
 successful_calls : set, unsuccessful_calls : set) -> None:
@@ -215,35 +262,11 @@ def rego_json_to_html(test_results_data : str, product : list, out_path : str,
                     RuntimeWarning)
             else:
                 for test in tests:
-                    # The following if/else makes the error handling backwards compatible until
-                    # all Regos are updated.
-                    if 'Prerequisites' not in test:
-                        prereqs = set()
-                    else:
-                        prereqs = set(test['Prerequisites'])
-                    # A call is failed if it is either missing from the successful_calls set
-                    # or present in the unsuccessful_calls
-                    failed_calls = set().union(
-                        prereqs.difference(successful_calls),
-                        prereqs.intersection(unsuccessful_calls)
-                    )
-                    if len(failed_calls) > 0:
+                    failed_prereqs = get_failed_prereqs(test, successful_calls, unsuccessful_calls)
+                    if len(failed_prereqs) > 0:
                         result = "Error"
                         report_stats["Error"] += 1
-                        failed_api_links = [
-                            API_LINKS[api] for api in failed_calls if api in API_LINKS
-                        ]
-                        failed_functions = [call for call in failed_calls if call not in API_LINKS]
-                        failed_details = ""
-                        if len(failed_api_links) > 0:
-                            links = ', '.join(failed_api_links)
-                            failed_details += f"This test depends on the following API call(s) " \
-                                f"which did not execute successfully: {links}. " \
-                                "See terminal output for more details. "
-                        if len(failed_functions) > 0:
-                            failed_details += f"This test depends on the following function(s) " \
-                                f"which did not execute successfully: {', '.join(failed_functions)}." \
-                                "See terminal output for more details."
+                        failed_details = get_failed_details(failed_prereqs)
                         table_data.append({
                             'Control ID': control['Id'],
                             'Requirement': control['Value'],

From f5514b84656f445f98ca18b244a231c52969a222 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 25 Jan 2024 08:30:33 -0800
Subject: [PATCH 17/35] correct error with f string

---
 scubagoggles/provider.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index c0bd01c0..370b8384 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -516,8 +516,8 @@ def call_gws_providers(self, products: list, services, quiet) -> dict:
                 )
                 self.successful_calls.add("reports/v1/activity/list")
         except Exception as exc:
-            warnings.warn(f"Provider Exception thrown while getting the logs; "\
-                "outputs will be incorrect: {exc}", RuntimeWarning)
+            warnings.warn("Provider Exception thrown while getting the logs; "\
+                f"outputs will be incorrect: {exc}", RuntimeWarning)
             self.unsuccessful_calls.add("reports/v1/activity/list")
 
         # repacks the main aggregator into the original form

From 7fc84ca3d627b9999ca56c60d68dd383a5ee5e60 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 25 Jan 2024 09:02:47 -0800
Subject: [PATCH 18/35] Add special case prereqs to rego files

---
 rego/Gmail.rego          | 6 ++++++
 rego/Groups.rego         | 2 ++
 scubagoggles/provider.py | 2 ++
 3 files changed, 10 insertions(+)

diff --git a/rego/Gmail.rego b/rego/Gmail.rego
index 63fc6f61..b98233e7 100644
--- a/rego/Gmail.rego
+++ b/rego/Gmail.rego
@@ -87,6 +87,7 @@ DomainsWithDkim contains DkimRecord.domain if {
 
 tests contains {
     "PolicyId": "GWS.GMAIL.2.1v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "get_dkim_records"],
     "Criticality": "Should",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutDkim, AllDomains),
     "ActualValue": input.dkim_records,
@@ -129,6 +130,7 @@ DomainsWithSpf contains SpfRecord.domain if {
 
 tests contains {
     "PolicyId": "GWS.GMAIL.3.2v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "get_spf_records"],
     "Criticality": "Shall",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutSpf, AllDomains),
     "ActualValue": DomainsWithoutSpf,
@@ -157,6 +159,7 @@ DomainsWithDmarc contains DmarcRecord.domain if {
 
 tests contains {
     "PolicyId": "GWS.GMAIL.4.1v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "get_dmarc_records"],
     "Criticality": "Shall",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutDmarc, AllDomains),
     "ActualValue": input.dmarc_records,
@@ -180,6 +183,7 @@ DomainsWithPreject contains DmarcRecord.domain if {
 
 tests contains {
     "PolicyId": "GWS.GMAIL.4.2v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "get_dmarc_records"],
     "Criticality": "Shall",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutPreject, AllDomains),
     "ActualValue": input.dmarc_records,
@@ -203,6 +207,7 @@ DomainsWithDHSContact contains DmarcRecord.domain if {
 
 tests contains {
     "PolicyId": "GWS.GMAIL.4.3v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "get_dmarc_records"],
     "Criticality": "Shall",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutDHSContact, AllDomains),
     "ActualValue": input.dmarc_records,
@@ -226,6 +231,7 @@ DomainsWithAgencyContact contains DmarcRecord.domain if {
 
 tests contains {
     "PolicyId": "GWS.GMAIL.4.4v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "get_dmarc_records"],
     "Criticality": "Should",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutAgencyContact, AllDomains),
     "ActualValue": input.dmarc_records,
diff --git a/rego/Groups.rego b/rego/Groups.rego
index 621838a5..f619e266 100644
--- a/rego/Groups.rego
+++ b/rego/Groups.rego
@@ -356,6 +356,7 @@ NonCompliantGroups7_1 contains Group.name if {
 # if there are no groups, it has to be safe.
 tests contains {
     "PolicyId": "GWS.GROUPS.7.1v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "directory/v1/groups/list"],
     "Criticality": "Should",
     "ReportDetails": NoGroupsDetails(Groups),
     "ActualValue": NoGroupsDetails(Groups),
@@ -370,6 +371,7 @@ if {
 # if there are groups
 tests contains {
     "PolicyId": "GWS.GROUPS.7.1v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "directory/v1/groups/list", "group-settings/v1/groups/get"],
     "Criticality": "Should",
     "ReportDetails": ReportDetailsGroups(NonCompliantGroups7_1),
     "ActualValue": {"NonCompliantGroups": NonCompliantGroups7_1},
diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index 370b8384..87544365 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -453,6 +453,7 @@ def get_group_settings(self, services) -> dict:
                 for group in response.get('groups'):
                     email = group.get('email')
                     group_settings.append(group_service.groups().get(groupUniqueId=email).execute())
+            self.successful_calls.add("directory/v1/groups/list")
             self.successful_calls.add("groups-settings/v1/groups/get")
             return {'group_settings': group_settings}
         except Exception as exc:
@@ -460,6 +461,7 @@ def get_group_settings(self, services) -> dict:
                 f"Exception thrown while getting group settings; outputs will be incorrect: {exc}",
                 RuntimeWarning
             )
+            self.unsuccessful_calls.add("directory/v1/groups/list")
             self.unsuccessful_calls.add("groups-settings/v1/groups/get")
             return {'group_settings': []}
 

From a6564e242495ca60772a0dd550426767f0a4af38 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 25 Jan 2024 09:25:17 -0800
Subject: [PATCH 19/35] Modify groups prereqs

---
 rego/Groups.rego | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rego/Groups.rego b/rego/Groups.rego
index f619e266..410ec4d3 100644
--- a/rego/Groups.rego
+++ b/rego/Groups.rego
@@ -356,7 +356,7 @@ NonCompliantGroups7_1 contains Group.name if {
 # if there are no groups, it has to be safe.
 tests contains {
     "PolicyId": "GWS.GROUPS.7.1v0.1",
-    "Prerequisites": ["directory/v1/domains/list", "directory/v1/groups/list"],
+    "Prerequisites": ["directory/v1/domains/list", "directory/v1/groups/list", "group-settings/v1/groups/get"],
     "Criticality": "Should",
     "ReportDetails": NoGroupsDetails(Groups),
     "ActualValue": NoGroupsDetails(Groups),

From f9f8280e437f3b1225b57e69ac89a6524958ed31 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 25 Jan 2024 09:25:45 -0800
Subject: [PATCH 20/35] Correct typo in reports API reference

---
 scubagoggles/provider.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index 87544365..828b12c7 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -516,11 +516,11 @@ def call_gws_providers(self, products: list, services, quiet) -> dict:
                         event=event
                     )
                 )
-                self.successful_calls.add("reports/v1/activity/list")
+                self.successful_calls.add("reports/v1/activities/list")
         except Exception as exc:
             warnings.warn("Provider Exception thrown while getting the logs; "\
                 f"outputs will be incorrect: {exc}", RuntimeWarning)
-            self.unsuccessful_calls.add("reports/v1/activity/list")
+            self.unsuccessful_calls.add("reports/v1/activities/list")
 
         # repacks the main aggregator into the original form
         # that the api returns the data in; under an 'items' key.

From cfdc934218e59867859278f617b72233931dbb57 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 25 Jan 2024 09:26:15 -0800
Subject: [PATCH 21/35] If prereqs not defined, assume dependence on reports
 api

---
 scubagoggles/reporter/reporter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index 2a42e79a..5e2f3e67 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -151,10 +151,10 @@ def get_failed_prereqs(test : dict, successful_calls : set, unsuccessful_calls :
     :param successful_calls: a set with the successful provider calls
     :param unsuccessful_calls: a set with the unsuccessful provider calls
     '''
-    # The following if/else makes the error handling backwards compatible until
-    # all Regos are updated.
     if 'Prerequisites' not in test:
-        prereqs = set()
+        # If Prerequisites is not defined, assume the test just depends on the
+        # reports API.
+        prereqs = set(["reports/v1/activities/list"])
     else:
         prereqs = set(test['Prerequisites'])
 

From 22832e5536d1afceecdc7914e8936579512ca3c7 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 30 Jan 2024 08:24:08 -0800
Subject: [PATCH 22/35] Fix error in provider from merge

---
 scubagoggles/provider.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index b915401f..e07c9dd8 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -547,7 +547,7 @@ def call_gws_providers(self, products: list, services, quiet, customer_id) -> di
                 product_to_items.update(self.get_super_admins(services['directory'], customer_id))
 
             if 'groups' in product_to_logs:
-                product_to_items.update(self.get_group_settings(services=services, customer_id))
+                product_to_items.update(self.get_group_settings(services, customer_id))
 
         except Exception as exc:
             warnings.warn(

From 88a752c1b606d1a99041143d67984e97bcc96b5d Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 30 Jan 2024 08:30:55 -0800
Subject: [PATCH 23/35] Add missing tests for gws 18

---
 rego/Gmail.rego | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/rego/Gmail.rego b/rego/Gmail.rego
index ed7ae32d..2cbc0954 100644
--- a/rego/Gmail.rego
+++ b/rego/Gmail.rego
@@ -1756,4 +1756,34 @@ tests contains {
     "RequirementMet": false,
     "NoSuchEvent": false
 }
+#--
+
+#
+# Baseline GWS.GMAIL.18.2v0.1
+#--
+# At this time we are unable to test because settings are configured in the GWS Admin Console
+# and not available within the generated logs
+tests contains {
+    "PolicyId": "GWS.GMAIL.18.2v0.1",
+    "Criticality": "Should/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false
+}
+#--
+
+#
+# Baseline GWS.GMAIL.18.3v0.1
+#--
+# At this time we are unable to test because settings are configured in the GWS Admin Console
+# and not available within the generated logs
+tests contains {
+    "PolicyId": "GWS.GMAIL.18.3v0.1",
+    "Criticality": "Shall/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false
+}
 #--
\ No newline at end of file

From 20842ffb6c3fd0bfba330d9897249739254e9d3b Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 30 Jan 2024 08:44:16 -0800
Subject: [PATCH 24/35] Fix other regressions caused by merge

---
 rego/Groups.rego                  |  4 ++--
 scubagoggles/provider.py          |  2 +-
 scubagoggles/reporter/reporter.py | 14 ++++++++------
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/rego/Groups.rego b/rego/Groups.rego
index 410ec4d3..ab3b223d 100644
--- a/rego/Groups.rego
+++ b/rego/Groups.rego
@@ -356,7 +356,7 @@ NonCompliantGroups7_1 contains Group.name if {
 # if there are no groups, it has to be safe.
 tests contains {
     "PolicyId": "GWS.GROUPS.7.1v0.1",
-    "Prerequisites": ["directory/v1/domains/list", "directory/v1/groups/list", "group-settings/v1/groups/get"],
+    "Prerequisites": ["directory/v1/domains/list", "directory/v1/groups/list", "groups-settings/v1/groups/get"],
     "Criticality": "Should",
     "ReportDetails": NoGroupsDetails(Groups),
     "ActualValue": NoGroupsDetails(Groups),
@@ -371,7 +371,7 @@ if {
 # if there are groups
 tests contains {
     "PolicyId": "GWS.GROUPS.7.1v0.1",
-    "Prerequisites": ["directory/v1/domains/list", "directory/v1/groups/list", "group-settings/v1/groups/get"],
+    "Prerequisites": ["directory/v1/domains/list", "directory/v1/groups/list", "groups-settings/v1/groups/get"],
     "Criticality": "Should",
     "ReportDetails": ReportDetailsGroups(NonCompliantGroups7_1),
     "ActualValue": {"NonCompliantGroups": NonCompliantGroups7_1},
diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index e07c9dd8..e83439b0 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -338,7 +338,7 @@ def get_tenant_info(self, service, customer_id) -> dict:
                     primary_domain = domain['domainName']
             return {
                 'domain': primary_domain,
-                'topLevelOU': self.get_toplevel_ou(service)
+                'topLevelOU': self.get_toplevel_ou(service, customer_id)
             }
         except Exception as exc:
             warnings.warn(
diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index 5e2f3e67..62b4f998 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -173,12 +173,14 @@ def get_reference_a_tag(api_call : str) -> str:
     
     :param api_call: a string representing the API call, such as "directory/v1/users/list".
     '''
-    api = api_call.split('/')[0]
-    call = '/'.join(api_call.split('/')[1:])
-    # All APIs except for the groups-settings api have "rest/" after "reference/"
-    api_type = "" if api == "groups-settings" else "rest/"
-    return f'<a href="https://developers.google.com/admin-sdk/{api}/reference/{api_type}{call}"> \
-        {api_call}</a>'
+    if api_call == "groups-settings/v1/groups/get":
+        # The reference URL for this API is structured differently than the others
+        return "https://developers.google.com/admin-sdk/groups-settings/v1/reference/groups/get"
+    else:
+        api = api_call.split('/')[0]
+        call = '/'.join(api_call.split('/')[1:])
+        return f'<a href="https://developers.google.com/admin-sdk/{api}/reference/rest/{call}"> \
+            {api_call}</a>'
 
 def get_failed_details(failed_prereqs : set) -> str:
     '''

From 6eff1b8c377896d2454968b423b21cd8c2de1441 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 30 Jan 2024 08:46:22 -0800
Subject: [PATCH 25/35] Make the linter happy

---
 scubagoggles/reporter/reporter.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index 62b4f998..4d292700 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -176,11 +176,10 @@ def get_reference_a_tag(api_call : str) -> str:
     if api_call == "groups-settings/v1/groups/get":
         # The reference URL for this API is structured differently than the others
         return "https://developers.google.com/admin-sdk/groups-settings/v1/reference/groups/get"
-    else:
-        api = api_call.split('/')[0]
-        call = '/'.join(api_call.split('/')[1:])
-        return f'<a href="https://developers.google.com/admin-sdk/{api}/reference/rest/{call}"> \
-            {api_call}</a>'
+    api = api_call.split('/')[0]
+    call = '/'.join(api_call.split('/')[1:])
+    return f'<a href="https://developers.google.com/admin-sdk/{api}/reference/rest/{call}"> \
+        {api_call}</a>'
 
 def get_failed_details(failed_prereqs : set) -> str:
     '''

From d553e88f8a1b4e7eda630f0b52e0160bcf488db8 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 30 Jan 2024 08:48:22 -0800
Subject: [PATCH 26/35] Refactor long lines

---
 scubagoggles/provider.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index e83439b0..cddfd2a6 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -495,7 +495,8 @@ def call_gws_providers(self, products: list, services, quiet, customer_id) -> di
             # Add top level organization unit name
             ou_ids.add(self.get_toplevel_ou(services['directory'], customer_id))
             # get all organizational unit data
-            product_to_items['organizational_units'] = self.get_ous(services['directory'], customer_id)
+            product_to_items['organizational_units'] = self.get_ous(services['directory'],
+                customer_id)
             for orgunit in product_to_items['organizational_units']['organizationUnits']:
                 ou_ids.add(orgunit['name'])
             # add just organizational unit names to a field]
@@ -538,7 +539,8 @@ def call_gws_providers(self, products: list, services, quiet, customer_id) -> di
                 product_to_items[key_name] = {'items': logs}
 
             # get tenant metadata for report front page header
-            product_to_items['tenant_info'] = self.get_tenant_info(services['directory'], customer_id)
+            product_to_items['tenant_info'] = self.get_tenant_info(services['directory'],
+                customer_id)
 
             if 'gmail' in product_to_logs: # add dns info if gmail is being run
                 product_to_items.update(self.get_dnsinfo(services['directory'], customer_id))

From 5a301aefa945800ef3cd463eca9c45c8e5864d4d Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Tue, 30 Jan 2024 09:01:43 -0800
Subject: [PATCH 27/35] Fix groups API reference links

---
 scubagoggles/reporter/reporter.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index 4d292700..d9b837a5 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -175,11 +175,12 @@ def get_reference_a_tag(api_call : str) -> str:
     '''
     if api_call == "groups-settings/v1/groups/get":
         # The reference URL for this API is structured differently than the others
-        return "https://developers.google.com/admin-sdk/groups-settings/v1/reference/groups/get"
+        return '<a href="https://developers.google.com/admin-sdk/groups-settings/v1/reference/' \
+            f'groups/get">{api_call}</a>'
     api = api_call.split('/')[0]
     call = '/'.join(api_call.split('/')[1:])
-    return f'<a href="https://developers.google.com/admin-sdk/{api}/reference/rest/{call}"> \
-        {api_call}</a>'
+    return f'<a href="https://developers.google.com/admin-sdk/{api}/reference/rest/{call}">' \
+        f'{api_call}</a>'
 
 def get_failed_details(failed_prereqs : set) -> str:
     '''
@@ -196,7 +197,7 @@ def get_failed_details(failed_prereqs : set) -> str:
             "directory/v1/domains/list",
             "directory/v1/groups/list",
             "reports/v1/activities/list",
-            "group-settings/v1/groups/get"
+            "groups-settings/v1/groups/get"
         ]
     }
 

From fc3f4769656dcf1a147db6bf0dc137aa47e24cd3 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 1 Feb 2024 10:10:29 -0800
Subject: [PATCH 28/35] Refactor services object to a state variable

---
 scubagoggles/orchestrator.py |  4 +-
 scubagoggles/provider.py     | 79 ++++++++++++++++--------------------
 2 files changed, 37 insertions(+), 46 deletions(-)

diff --git a/scubagoggles/orchestrator.py b/scubagoggles/orchestrator.py
index 87e1bfc2..7e646501 100644
--- a/scubagoggles/orchestrator.py
+++ b/scubagoggles/orchestrator.py
@@ -63,8 +63,8 @@ def run_gws_providers(args, services):
     out_folder = args.outputpath
     provider_dict = {}
 
-    provider = Provider()
-    provider_dict = provider.call_gws_providers(products, services, args.quiet, args.customerid)
+    provider = Provider(services)
+    provider_dict = provider.call_gws_providers(products, args.quiet, args.customerid)
     provider_dict['successful_calls'] = list(provider.successful_calls)
     provider_dict['unsuccessful_calls'] = list(provider.unsuccessful_calls)
 
diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index cddfd2a6..67f17303 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -91,9 +91,15 @@ class Provider:
     Class for making the GWS api calls and tracking the results.
     '''
 
-    def __init__(self):
+    def __init__(self, services : dict):
+        '''
+        Initialize the Provider.
+
+        :param services: a dict of service objects.
+        '''
         self.successful_calls = set()
         self.unsuccessful_calls = set()
+        self.services = services
 
     def get_spf_records(self, domains: list) -> list:
         '''
@@ -192,17 +198,16 @@ def get_dmarc_records(self, domains : list) -> list:
     See ProviderSettingsExport.json under 'dmarc_records' for more details.", RuntimeWarning)
         return results
 
-    def get_dnsinfo(self, service, customer_id):
+    def get_dnsinfo(self,customer_id):
         '''
         Gets DNS Information for Gmail baseline
 
-        :param service: a directory_v1 service instance
         :param customer_id: the ID of the customer to run against
         '''
         output = {"domains": [], "spf_records": [], "dkim_records": [], "dmarc_records": []}
 
         # Determine the tenant's domains via the API
-        response = service.domains().list(customer=customer_id).execute()
+        response = self.services['directory'].domains().list(customer=customer_id).execute()
         domains = {d['domainName'] for d in response['domains']}
 
         if len(domains) == 0:
@@ -234,15 +239,15 @@ def get_dnsinfo(self, service, customer_id):
             self.unsuccessful_calls.add("get_dmarc_records")
         return output
 
-    def get_super_admins(self, service, customer_id) -> dict:
+    def get_super_admins(self, customer_id) -> dict:
         '''
         Gets the org unit/primary email of all super admins, using the directory API
 
-        :param service: a directory_v1 service instance
         :param customer_id: the ID of the customer to run against
         '''
         try:
-            response = service.users().list(customer=customer_id, query="isAdmin=True").execute()
+            response = self.services['directory'].users()\
+                .list(customer=customer_id, query="isAdmin=True").execute()
             admins = []
             for user in response['users']:
                 org_unit = user['orgUnitPath']
@@ -260,16 +265,15 @@ def get_super_admins(self, service, customer_id) -> dict:
             self.unsuccessful_calls.add("directory/v1/users/list")
             return {'super_admins': []}
 
-    def get_ous(self, service, customer_id) -> dict:
+    def get_ous(self, customer_id) -> dict:
         '''
         Gets the organizational units using the directory API
 
-        :param service: a directory_v1 service instance
         :param customer_id: the ID of the customer to run against
         '''
 
         try:
-            response = service.orgunits().list(customerId=customer_id).execute()
+            response = self.services['directory'].orgunits().list(customerId=customer_id).execute()
             self.successful_calls.add("directory/v1/orgunits/list")
             if 'organizationUnits' not in response:
                 return {}
@@ -282,18 +286,16 @@ def get_ous(self, service, customer_id) -> dict:
             self.unsuccessful_calls.add("directory/v1/orgunits/list")
             return {}
 
-    def get_toplevel_ou(self, service, customer_id) -> str:
+    def get_toplevel_ou(self, customer_id) -> str:
         '''
         Gets the tenant name using the directory API
 
-        :param service: a directory_v1 service instance
         :param customer_id: the ID of the customer to run against
         '''
 
         try:
-            response = service.orgunits().list(customerId=customer_id,
-                                        orgUnitPath='/',
-                                        type='children').execute()
+            response = self.services['directory'].orgunits()\
+                .list(customerId=customer_id, orgUnitPath='/', type='children').execute()
             # Because we set orgUnitPath to / and type to children, the API call will only
             # return the second-level OUs, meaning the parentOrgUnitId of any of the OUs returned
             # will point us to OU of the entire organization
@@ -308,8 +310,8 @@ def get_toplevel_ou(self, service, customer_id) -> str:
                 # changes have to apply to the top-level OU.
                 return ""
             parent_ou = response['organizationUnits'][0]['parentOrgUnitId']
-            response = service.orgunits().get(customerId=customer_id, orgUnitPath=parent_ou)\
-                .execute()
+            response = self.services['directory'].orgunits()\
+                .get(customerId=customer_id, orgUnitPath=parent_ou).execute()
             ou_name = response['name']
             self.successful_calls.add("directory/v1/orgunits/list")
             return ou_name
@@ -322,15 +324,14 @@ def get_toplevel_ou(self, service, customer_id) -> str:
             return ""
 
 
-    def get_tenant_info(self, service, customer_id) -> dict:
+    def get_tenant_info(self, customer_id) -> dict:
         '''
         Gets the high-level tenant info using the directory API
 
-        :param service: a directory_v1 service instance
         :param customer_id: the ID of the customer to run against
         '''
         try:
-            response = service.domains().list(customer=customer_id).execute()
+            response = self.services['directory'].domains().list(customer=customer_id).execute()
             self.successful_calls.add("directory/v1/domains/list")
             primary_domain = ""
             for domain in response['domains']:
@@ -338,7 +339,7 @@ def get_tenant_info(self, service, customer_id) -> dict:
                     primary_domain = domain['domainName']
             return {
                 'domain': primary_domain,
-                'topLevelOU': self.get_toplevel_ou(service, customer_id)
+                'topLevelOU': self.get_toplevel_ou(customer_id)
             }
         except Exception as exc:
             warnings.warn(
@@ -352,7 +353,7 @@ def get_tenant_info(self, service, customer_id) -> dict:
             }
 
 
-    def get_gws_logs(self, products: list, service, event: str) -> dict:
+    def get_gws_logs(self, products: list, event: str) -> dict:
         '''
         Gets the GWS admin audit logs with the specified event name.
         This function will also some parsing and filtering to ensure that an appropriate
@@ -361,13 +362,12 @@ def get_gws_logs(self, products: list, service, event: str) -> dict:
         across products in the resulting provider JSON.
 
         :param products: a narrowed list of the products being invoked
-        :param service: service is a Google reports API object, created from successfully
         authenticating in auth.py
         :param event: the name of the specific event we are querying for.
         '''
 
         # Filter responses by org_unit id
-        response = (service.activities().list(userKey='all',
+        response = (self.services['reports'].activities().list(userKey='all',
                 applicationName='admin',
                 eventName=event).execute()).get('items', [])
 
@@ -427,16 +427,15 @@ def get_gws_logs(self, products: list, service, event: str) -> dict:
             )
         return products_to_logs
 
-    def get_group_settings(self, services, customer_id) -> dict:
+    def get_group_settings(self, customer_id) -> dict:
         '''
         Gets all of the group info using the directory API and group settings API
 
-        :param services: a service instance
         :param customer_id: the ID of the customer to run against
         '''
 
-        group_service = services['groups']
-        domain_service = services['directory']
+        group_service = self.services['groups']
+        domain_service = self.services['directory']
         try:
             # gather all of the domains within a suite to get groups
             response = domain_service.domains().list(customer=customer_id).execute()
@@ -471,16 +470,14 @@ def get_group_settings(self, services, customer_id) -> dict:
             self.unsuccessful_calls.add("groups-settings/v1/groups/get")
             return {'group_settings': []}
 
-    def call_gws_providers(self, products: list, services, quiet, customer_id) -> dict:
+    def call_gws_providers(self, products: list, quiet, customer_id) -> dict:
         '''
         Calls the relevant GWS APIs to get the data we need for the baselines.
         Data such as the admin audit log, super admin users etc.
 
         :param products: list of product names to check
-        :param services: a dict of service objects.
         :param quiet: suppress tqdm output
         :param customer_id: the ID of the customer to run against
-        service is a Google reports API object, created from successfully authenticating in auth.py
         '''
         # create a inverse dictionary containing a mapping of event => list of products
         events_to_products = create_subset_inverted_dict(EVENTS, products)
@@ -493,10 +490,9 @@ def call_gws_providers(self, products: list, services, quiet, customer_id) -> di
         ou_ids.add("") # certain settings have no OU
         try:
             # Add top level organization unit name
-            ou_ids.add(self.get_toplevel_ou(services['directory'], customer_id))
+            ou_ids.add(self.get_toplevel_ou(customer_id))
             # get all organizational unit data
-            product_to_items['organizational_units'] = self.get_ous(services['directory'],
-                customer_id)
+            product_to_items['organizational_units'] = self.get_ous(customer_id)
             for orgunit in product_to_items['organizational_units']['organizationUnits']:
                 ou_ids.add(orgunit['name'])
             # add just organizational unit names to a field]
@@ -518,11 +514,7 @@ def call_gws_providers(self, products: list, services, quiet, customer_id) -> di
                 # aggregator dict
                 product_to_logs = merge_dicts(
                     product_to_logs,
-                    self.get_gws_logs(
-                        products=product_list,
-                        service=services['reports'],
-                        event=event
-                    )
+                    self.get_gws_logs(products=product_list, event=event)
                 )
                 self.successful_calls.add("reports/v1/activities/list")
         except Exception as exc:
@@ -539,17 +531,16 @@ def call_gws_providers(self, products: list, services, quiet, customer_id) -> di
                 product_to_items[key_name] = {'items': logs}
 
             # get tenant metadata for report front page header
-            product_to_items['tenant_info'] = self.get_tenant_info(services['directory'],
-                customer_id)
+            product_to_items['tenant_info'] = self.get_tenant_info(customer_id)
 
             if 'gmail' in product_to_logs: # add dns info if gmail is being run
-                product_to_items.update(self.get_dnsinfo(services['directory'], customer_id))
+                product_to_items.update(self.get_dnsinfo(customer_id))
 
             if 'commoncontrols' in product_to_logs: # add list of super admins if CC is being run
-                product_to_items.update(self.get_super_admins(services['directory'], customer_id))
+                product_to_items.update(self.get_super_admins(customer_id))
 
             if 'groups' in product_to_logs:
-                product_to_items.update(self.get_group_settings(services, customer_id))
+                product_to_items.update(self.get_group_settings(customer_id))
 
         except Exception as exc:
             warnings.warn(

From 918400e98595be3cdbc21e9bbb6355e5c8ccd874 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 1 Feb 2024 10:25:35 -0800
Subject: [PATCH 29/35] Make customer_id and dnsclient state variables

---
 scubagoggles/orchestrator.py |  8 ++--
 scubagoggles/provider.py     | 79 ++++++++++++++++--------------------
 2 files changed, 38 insertions(+), 49 deletions(-)

diff --git a/scubagoggles/orchestrator.py b/scubagoggles/orchestrator.py
index 7e646501..e7312502 100644
--- a/scubagoggles/orchestrator.py
+++ b/scubagoggles/orchestrator.py
@@ -51,20 +51,20 @@ def gws_products() -> dict:
     }
     return gws
 
-def run_gws_providers(args, services):
+def run_gws_providers(args, services : dict):
     """
     Runs the provider scripts and outputs a json to path
 
     :param args: the command line arguments to this script
-    :param services: a Google API services object
+    :param services: a dictionary of Google API service objects
     """
 
     products = args.baselines
     out_folder = args.outputpath
     provider_dict = {}
 
-    provider = Provider(services)
-    provider_dict = provider.call_gws_providers(products, args.quiet, args.customerid)
+    provider = Provider(services, args.customerid)
+    provider_dict = provider.call_gws_providers(products, args.quiet)
     provider_dict['successful_calls'] = list(provider.successful_calls)
     provider_dict['unsuccessful_calls'] = list(provider.unsuccessful_calls)
 
diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index 67f17303..d4437062 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -66,7 +66,7 @@
     'all': [None]
 }
 
-selectors = ["google", "selector1", "selector2"]
+SELECTORS = ["google", "selector1", "selector2"]
 # For DKIM.
 # Unfortunately, hard-coded. Ideally, we'd be able to use an API to get
 # the selectors used programmatically, but it doesn't seem like there is
@@ -84,22 +84,23 @@
 #    beginning of the domain name up to the first period
 #
 
-DNSClient = RobustDNSClient()
-
 class Provider:
     '''
     Class for making the GWS api calls and tracking the results.
     '''
 
-    def __init__(self, services : dict):
+    def __init__(self, services : dict, customer_id : str):
         '''
         Initialize the Provider.
 
         :param services: a dict of service objects.
+        :param customer_id: the ID of the customer to run against.
         '''
+        self.services = services
+        self.customer_id = customer_id
         self.successful_calls = set()
         self.unsuccessful_calls = set()
-        self.services = services
+        self.dns_client = RobustDNSClient()
 
     def get_spf_records(self, domains: list) -> list:
         '''
@@ -110,7 +111,7 @@ def get_spf_records(self, domains: list) -> list:
         results = []
         n_low_confidence = 0
         for domain in domains:
-            result = DNSClient.query(domain)
+            result = self.dns_client.query(domain)
             if not result['HighConfidence']:
                 n_low_confidence += 1
             results.append({
@@ -135,10 +136,10 @@ def get_dkim_records(self, domains : list) -> list:
         results = []
         n_low_confidence = 0
         for domain in domains:
-            qnames = [f"{selector}._domainkey.{domain}" for selector in selectors]
+            qnames = [f"{selector}._domainkey.{domain}" for selector in SELECTORS]
             log_entries = []
             for qname in qnames:
-                result = DNSClient.query(qname)
+                result = self.dns_client.query(qname)
                 log_entries.extend(result['LogEntries'])
                 if len(result['Answers']) == 0:
                     # The DKIM record does not exist with this selector, we need to try again with
@@ -174,14 +175,14 @@ def get_dmarc_records(self, domains : list) -> list:
         for domain in domains:
             log_entries = []
             qname = f"_dmarc.{domain}"
-            result = DNSClient.query(qname)
+            result = self.dns_client.query(qname)
             log_entries.extend(result['LogEntries'])
             if len(result["Answers"]) == 0:
                 # The domain does not exist. If the record is not available at the full domain
                 # level, we need to check at the organizational domain level.
                 labels = domain.split(".")
                 org_domain = f"{labels[-2]}.{labels[-1]}"
-                result = DNSClient.query(f"_dmarc.{org_domain}")
+                result = self.dns_client.query(f"_dmarc.{org_domain}")
                 log_entries.extend(result['LogEntries'])
             if not result['HighConfidence']:
                 n_low_confidence += 1
@@ -198,16 +199,14 @@ def get_dmarc_records(self, domains : list) -> list:
     See ProviderSettingsExport.json under 'dmarc_records' for more details.", RuntimeWarning)
         return results
 
-    def get_dnsinfo(self,customer_id):
+    def get_dnsinfo(self):
         '''
         Gets DNS Information for Gmail baseline
-
-        :param customer_id: the ID of the customer to run against
         '''
         output = {"domains": [], "spf_records": [], "dkim_records": [], "dmarc_records": []}
 
         # Determine the tenant's domains via the API
-        response = self.services['directory'].domains().list(customer=customer_id).execute()
+        response = self.services['directory'].domains().list(customer=self.customer_id).execute()
         domains = {d['domainName'] for d in response['domains']}
 
         if len(domains) == 0:
@@ -239,15 +238,13 @@ def get_dnsinfo(self,customer_id):
             self.unsuccessful_calls.add("get_dmarc_records")
         return output
 
-    def get_super_admins(self, customer_id) -> dict:
+    def get_super_admins(self) -> dict:
         '''
         Gets the org unit/primary email of all super admins, using the directory API
-
-        :param customer_id: the ID of the customer to run against
         '''
         try:
             response = self.services['directory'].users()\
-                .list(customer=customer_id, query="isAdmin=True").execute()
+                .list(customer=self.customer_id, query="isAdmin=True").execute()
             admins = []
             for user in response['users']:
                 org_unit = user['orgUnitPath']
@@ -265,15 +262,14 @@ def get_super_admins(self, customer_id) -> dict:
             self.unsuccessful_calls.add("directory/v1/users/list")
             return {'super_admins': []}
 
-    def get_ous(self, customer_id) -> dict:
+    def get_ous(self) -> dict:
         '''
         Gets the organizational units using the directory API
-
-        :param customer_id: the ID of the customer to run against
         '''
 
         try:
-            response = self.services['directory'].orgunits().list(customerId=customer_id).execute()
+            response = self.services['directory'].orgunits().list(customerId=self.customer_id)\
+                .execute()
             self.successful_calls.add("directory/v1/orgunits/list")
             if 'organizationUnits' not in response:
                 return {}
@@ -286,16 +282,14 @@ def get_ous(self, customer_id) -> dict:
             self.unsuccessful_calls.add("directory/v1/orgunits/list")
             return {}
 
-    def get_toplevel_ou(self, customer_id) -> str:
+    def get_toplevel_ou(self) -> str:
         '''
         Gets the tenant name using the directory API
-
-        :param customer_id: the ID of the customer to run against
         '''
 
         try:
             response = self.services['directory'].orgunits()\
-                .list(customerId=customer_id, orgUnitPath='/', type='children').execute()
+                .list(customerId=self.customer_id, orgUnitPath='/', type='children').execute()
             # Because we set orgUnitPath to / and type to children, the API call will only
             # return the second-level OUs, meaning the parentOrgUnitId of any of the OUs returned
             # will point us to OU of the entire organization
@@ -311,7 +305,7 @@ def get_toplevel_ou(self, customer_id) -> str:
                 return ""
             parent_ou = response['organizationUnits'][0]['parentOrgUnitId']
             response = self.services['directory'].orgunits()\
-                .get(customerId=customer_id, orgUnitPath=parent_ou).execute()
+                .get(customerId=self.customer_id, orgUnitPath=parent_ou).execute()
             ou_name = response['name']
             self.successful_calls.add("directory/v1/orgunits/list")
             return ou_name
@@ -324,14 +318,13 @@ def get_toplevel_ou(self, customer_id) -> str:
             return ""
 
 
-    def get_tenant_info(self, customer_id) -> dict:
+    def get_tenant_info(self) -> dict:
         '''
         Gets the high-level tenant info using the directory API
-
-        :param customer_id: the ID of the customer to run against
         '''
         try:
-            response = self.services['directory'].domains().list(customer=customer_id).execute()
+            response = self.services['directory'].domains().list(customer=self.customer_id)\
+                .execute()
             self.successful_calls.add("directory/v1/domains/list")
             primary_domain = ""
             for domain in response['domains']:
@@ -339,7 +332,7 @@ def get_tenant_info(self, customer_id) -> dict:
                     primary_domain = domain['domainName']
             return {
                 'domain': primary_domain,
-                'topLevelOU': self.get_toplevel_ou(customer_id)
+                'topLevelOU': self.get_toplevel_ou()
             }
         except Exception as exc:
             warnings.warn(
@@ -427,20 +420,17 @@ def get_gws_logs(self, products: list, event: str) -> dict:
             )
         return products_to_logs
 
-    def get_group_settings(self, customer_id) -> dict:
+    def get_group_settings(self) -> dict:
         '''
         Gets all of the group info using the directory API and group settings API
-
-        :param customer_id: the ID of the customer to run against
         '''
 
         group_service = self.services['groups']
         domain_service = self.services['directory']
         try:
             # gather all of the domains within a suite to get groups
-            response = domain_service.domains().list(customer=customer_id).execute()
+            response = domain_service.domains().list(customer=self.customer_id).execute()
             domains = {d['domainName'] for d in response['domains'] if d['verified']}
-
             self.successful_calls.add("directory/v1/domains/list")
         except Exception as exc:
             warnings.warn(
@@ -470,14 +460,13 @@ def get_group_settings(self, customer_id) -> dict:
             self.unsuccessful_calls.add("groups-settings/v1/groups/get")
             return {'group_settings': []}
 
-    def call_gws_providers(self, products: list, quiet, customer_id) -> dict:
+    def call_gws_providers(self, products: list, quiet) -> dict:
         '''
         Calls the relevant GWS APIs to get the data we need for the baselines.
         Data such as the admin audit log, super admin users etc.
 
         :param products: list of product names to check
         :param quiet: suppress tqdm output
-        :param customer_id: the ID of the customer to run against
         '''
         # create a inverse dictionary containing a mapping of event => list of products
         events_to_products = create_subset_inverted_dict(EVENTS, products)
@@ -490,9 +479,9 @@ def call_gws_providers(self, products: list, quiet, customer_id) -> dict:
         ou_ids.add("") # certain settings have no OU
         try:
             # Add top level organization unit name
-            ou_ids.add(self.get_toplevel_ou(customer_id))
+            ou_ids.add(self.get_toplevel_ou())
             # get all organizational unit data
-            product_to_items['organizational_units'] = self.get_ous(customer_id)
+            product_to_items['organizational_units'] = self.get_ous()
             for orgunit in product_to_items['organizational_units']['organizationUnits']:
                 ou_ids.add(orgunit['name'])
             # add just organizational unit names to a field]
@@ -531,16 +520,16 @@ def call_gws_providers(self, products: list, quiet, customer_id) -> dict:
                 product_to_items[key_name] = {'items': logs}
 
             # get tenant metadata for report front page header
-            product_to_items['tenant_info'] = self.get_tenant_info(customer_id)
+            product_to_items['tenant_info'] = self.get_tenant_info()
 
             if 'gmail' in product_to_logs: # add dns info if gmail is being run
-                product_to_items.update(self.get_dnsinfo(customer_id))
+                product_to_items.update(self.get_dnsinfo())
 
             if 'commoncontrols' in product_to_logs: # add list of super admins if CC is being run
-                product_to_items.update(self.get_super_admins(customer_id))
+                product_to_items.update(self.get_super_admins())
 
             if 'groups' in product_to_logs:
-                product_to_items.update(self.get_group_settings(customer_id))
+                product_to_items.update(self.get_group_settings())
 
         except Exception as exc:
             warnings.warn(

From 0774d30662b902a41ce10125cfd672a07266cb5e Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 1 Feb 2024 11:07:33 -0800
Subject: [PATCH 30/35] Ensure the domains() API is called only once

---
 scubagoggles/provider.py | 72 ++++++++++++++++------------------------
 1 file changed, 29 insertions(+), 43 deletions(-)

diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index d4437062..3a77294f 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -101,7 +101,23 @@ def __init__(self, services : dict, customer_id : str):
         self.successful_calls = set()
         self.unsuccessful_calls = set()
         self.dns_client = RobustDNSClient()
+        self.domains = None
 
+    def list_domains(self) -> list:
+        '''
+        Return the customer's domains. Ensures that the domains API is called only once and that
+        the domains used throughout the provider are consistent.
+        '''
+        if self.domains is None:
+            try:
+                self.domains = self.services['directory'].domains().list(customer=self.customer_id)\
+                    .execute()['domains']
+                self.successful_calls.add("directory/v1/domains/list")
+            except:
+                self.domains = []
+                self.unsuccessful_calls.add("directory/v1/domains/list")
+        return self.domains
+        
     def get_spf_records(self, domains: list) -> list:
         '''
         Gets the SPF records for each domain in domains.
@@ -204,11 +220,7 @@ def get_dnsinfo(self):
         Gets DNS Information for Gmail baseline
         '''
         output = {"domains": [], "spf_records": [], "dkim_records": [], "dmarc_records": []}
-
-        # Determine the tenant's domains via the API
-        response = self.services['directory'].domains().list(customer=self.customer_id).execute()
-        domains = {d['domainName'] for d in response['domains']}
-
+        domains = {d['domainName'] for d in self.list_domains()}
         if len(domains) == 0:
             warnings.warn("No domains found.", RuntimeWarning)
             return output
@@ -315,36 +327,21 @@ def get_toplevel_ou(self) -> str:
                 RuntimeWarning
             )
             self.unsuccessful_calls.add("directory/v1/orgunits/list")
-            return ""
+            return "Error Retrieving"
 
 
     def get_tenant_info(self) -> dict:
         '''
         Gets the high-level tenant info using the directory API
         '''
-        try:
-            response = self.services['directory'].domains().list(customer=self.customer_id)\
-                .execute()
-            self.successful_calls.add("directory/v1/domains/list")
-            primary_domain = ""
-            for domain in response['domains']:
-                if domain['isPrimary']:
-                    primary_domain = domain['domainName']
-            return {
-                'domain': primary_domain,
-                'topLevelOU': self.get_toplevel_ou()
-            }
-        except Exception as exc:
-            warnings.warn(
-                f"An exception was thrown trying to get the tenant info: {exc}",
-                RuntimeWarning
-            )
-            self.unsuccessful_calls.add("directory/v1/domains/list")
-            return {
-                'domain': 'Error Retrieving',
-                'topLevelOU': 'Error Retrieving'
-            }
-
+        primary_domain = "Error Retrieving"
+        for domain in self.list_domains():
+            if domain['isPrimary']:
+                primary_domain = domain['domainName']
+        return {
+            'domain': primary_domain,
+            'topLevelOU': self.get_toplevel_ou()
+        }
 
     def get_gws_logs(self, products: list, event: str) -> dict:
         '''
@@ -426,25 +423,14 @@ def get_group_settings(self) -> dict:
         '''
 
         group_service = self.services['groups']
-        domain_service = self.services['directory']
-        try:
-            # gather all of the domains within a suite to get groups
-            response = domain_service.domains().list(customer=self.customer_id).execute()
-            domains = {d['domainName'] for d in response['domains'] if d['verified']}
-            self.successful_calls.add("directory/v1/domains/list")
-        except Exception as exc:
-            warnings.warn(
-                f"Exception thrown while getting group settings; outputs will be incorrect: {exc}",
-                RuntimeWarning
-            )
-            self.unsuccessful_calls.add("directory/v1/domains/list")
-            return {'group_settings': []}
+        directory_service = self.services['directory']
+        domains = {d['domainName'] for d in self.list_domains() if d['verified']}
 
         try:
             # get the group settings for each groups
             group_settings = []
             for domain in domains:
-                response = domain_service.groups().list(domain=domain).execute()
+                response = directory_service.groups().list(domain=domain).execute()
                 for group in response.get('groups'):
                     email = group.get('email')
                     group_settings.append(group_service.groups().get(groupUniqueId=email).execute())

From d5300aa601ab7d39311fee7fcdb90fa979358a73 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 1 Feb 2024 11:14:17 -0800
Subject: [PATCH 31/35] Changes to satisfy the linter

---
 scubagoggles/provider.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index 3a77294f..d8523dd6 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -113,11 +113,12 @@ def list_domains(self) -> list:
                 self.domains = self.services['directory'].domains().list(customer=self.customer_id)\
                     .execute()['domains']
                 self.successful_calls.add("directory/v1/domains/list")
-            except:
+            except Exception as exc:
                 self.domains = []
+                warnings.warn(f"An exception was thrown by list_domains: {exc}", RuntimeWarning)
                 self.unsuccessful_calls.add("directory/v1/domains/list")
         return self.domains
-        
+
     def get_spf_records(self, domains: list) -> list:
         '''
         Gets the SPF records for each domain in domains.

From 00c959c99f3151f5f576b591ae4e072547aa880c Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 1 Feb 2024 14:57:04 -0800
Subject: [PATCH 32/35] Convert API reference to enums

---
 scubagoggles/api_reference.py     | 23 +++++++++++++++++++++++
 scubagoggles/provider.py          | 30 ++++++++++++++++--------------
 scubagoggles/reporter/reporter.py | 30 +++---------------------------
 3 files changed, 42 insertions(+), 41 deletions(-)
 create mode 100644 scubagoggles/api_reference.py

diff --git a/scubagoggles/api_reference.py b/scubagoggles/api_reference.py
new file mode 100644
index 00000000..63968441
--- /dev/null
+++ b/scubagoggles/api_reference.py
@@ -0,0 +1,23 @@
+from enum import Enum
+
+BASE_URL = "https://developers.google.com/admin-sdk"
+
+class ApiReference(Enum):
+    LIST_USERS = "directory/v1/users/list"
+    LIST_OUS = "directory/v1/orgunits/list"
+    LIST_DOMAINS = "directory/v1/domains/list"
+    LIST_GROUPS = "directory/v1/groups/list"
+    LIST_ACTIVITIES = "reports/v1/activities/list"
+    GET_GROUP = "groups-settings/v1/groups/get"
+
+class ApiUrl(Enum):
+    LIST_USERS = f"{BASE_URL}/directory/reference/rest/v1/users/list"
+    LIST_OUS = f"{BASE_URL}/directory/reference/rest/v1/orgunits/list"
+    LIST_DOMAINS = f"{BASE_URL}/directory/reference/rest/v1/domains/list"
+    LIST_GROUPS = f"{BASE_URL}/directory/reference/rest/v1/groups/list"
+    LIST_ACTIVITIES = f"{BASE_URL}/reports/reference/rest/v1/activities/list"
+    GET_GROUP = f"{BASE_URL}/groups-settings/v1/reference/"
+
+API_LINKS = {
+    api.value: f'<a href="{ApiUrl[api.name]}">{api.value}</a>' for api in ApiReference
+}
\ No newline at end of file
diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index d8523dd6..5dd4e05f 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -6,6 +6,7 @@
 from tqdm import tqdm
 
 from scubagoggles.utils import create_subset_inverted_dict, create_key_to_list, merge_dicts
+from scubagoggles.api_reference import ApiReference
 from scubagoggles.robust_dns import RobustDNSClient
 
 EVENTS = {
@@ -66,6 +67,7 @@
     'all': [None]
 }
 
+
 SELECTORS = ["google", "selector1", "selector2"]
 # For DKIM.
 # Unfortunately, hard-coded. Ideally, we'd be able to use an API to get
@@ -112,11 +114,11 @@ def list_domains(self) -> list:
             try:
                 self.domains = self.services['directory'].domains().list(customer=self.customer_id)\
                     .execute()['domains']
-                self.successful_calls.add("directory/v1/domains/list")
+                self.successful_calls.add(ApiReference.LIST_DOMAINS.value)
             except Exception as exc:
                 self.domains = []
                 warnings.warn(f"An exception was thrown by list_domains: {exc}", RuntimeWarning)
-                self.unsuccessful_calls.add("directory/v1/domains/list")
+                self.unsuccessful_calls.add(ApiReference.LIST_DOMAINS.value)
         return self.domains
 
     def get_spf_records(self, domains: list) -> list:
@@ -265,14 +267,14 @@ def get_super_admins(self) -> dict:
                 org_unit = org_unit[1:] if org_unit.startswith('/') else org_unit
                 email = user['primaryEmail']
                 admins.append({'primaryEmail': email, 'orgUnitPath': org_unit})
-            self.successful_calls.add("directory/v1/users/list")
+            self.successful_calls.add(ApiReference.LIST_USERS.value)
             return {'super_admins': admins}
         except Exception as exc:
             warnings.warn(
                 f"Exception thrown while getting super admins; outputs will be incorrect: {exc}",
                 RuntimeWarning
             )
-            self.unsuccessful_calls.add("directory/v1/users/list")
+            self.unsuccessful_calls.add(ApiReference.LIST_USERS.value)
             return {'super_admins': []}
 
     def get_ous(self) -> dict:
@@ -283,7 +285,7 @@ def get_ous(self) -> dict:
         try:
             response = self.services['directory'].orgunits().list(customerId=self.customer_id)\
                 .execute()
-            self.successful_calls.add("directory/v1/orgunits/list")
+            self.successful_calls.add(ApiReference.LIST_OUS.value)
             if 'organizationUnits' not in response:
                 return {}
             return response
@@ -292,7 +294,7 @@ def get_ous(self) -> dict:
                 f"Exception thrown while getting top level OU: {exc}",
                 RuntimeWarning
             )
-            self.unsuccessful_calls.add("directory/v1/orgunits/list")
+            self.unsuccessful_calls.add(ApiReference.LIST_OUS.value)
             return {}
 
     def get_toplevel_ou(self) -> str:
@@ -320,14 +322,14 @@ def get_toplevel_ou(self) -> str:
             response = self.services['directory'].orgunits()\
                 .get(customerId=self.customer_id, orgUnitPath=parent_ou).execute()
             ou_name = response['name']
-            self.successful_calls.add("directory/v1/orgunits/list")
+            self.successful_calls.add(ApiReference.LIST_OUS.value)
             return ou_name
         except Exception as exc:
             warnings.warn(
                 f"Exception thrown while getting top level OU: {exc}",
                 RuntimeWarning
             )
-            self.unsuccessful_calls.add("directory/v1/orgunits/list")
+            self.unsuccessful_calls.add(ApiReference.LIST_OUS.value)
             return "Error Retrieving"
 
 
@@ -435,16 +437,16 @@ def get_group_settings(self) -> dict:
                 for group in response.get('groups'):
                     email = group.get('email')
                     group_settings.append(group_service.groups().get(groupUniqueId=email).execute())
-            self.successful_calls.add("directory/v1/groups/list")
-            self.successful_calls.add("groups-settings/v1/groups/get")
+            self.successful_calls.add(ApiReference.LIST_GROUPS.value)
+            self.successful_calls.add(ApiReference.GET_GROUP.value)
             return {'group_settings': group_settings}
         except Exception as exc:
             warnings.warn(
                 f"Exception thrown while getting group settings; outputs will be incorrect: {exc}",
                 RuntimeWarning
             )
-            self.unsuccessful_calls.add("directory/v1/groups/list")
-            self.unsuccessful_calls.add("groups-settings/v1/groups/get")
+            self.unsuccessful_calls.add(ApiReference.LIST_GROUPS.value)
+            self.unsuccessful_calls.add(ApiReference.GET_GROUP.value)
             return {'group_settings': []}
 
     def call_gws_providers(self, products: list, quiet) -> dict:
@@ -492,11 +494,11 @@ def call_gws_providers(self, products: list, quiet) -> dict:
                     product_to_logs,
                     self.get_gws_logs(products=product_list, event=event)
                 )
-                self.successful_calls.add("reports/v1/activities/list")
+                self.successful_calls.add(ApiReference.LIST_ACTIVITIES.value)
         except Exception as exc:
             warnings.warn("Provider Exception thrown while getting the logs; "\
                 f"outputs will be incorrect: {exc}", RuntimeWarning)
-            self.unsuccessful_calls.add("reports/v1/activities/list")
+            self.unsuccessful_calls.add(ApiReference.LIST_ACTIVITIES.value)
 
         # repacks the main aggregator into the original form
         # that the api returns the data in; under an 'items' key.
diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index d9b837a5..f5f83cd6 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -9,6 +9,7 @@
 from datetime import datetime
 import pandas as pd
 from scubagoggles.utils import rel_abs_path
+from scubagoggles.api_reference import API_LINKS
 
 SCUBA_GITHUB_URL = "https://github.com/cisagov/scubagoggles"
 
@@ -167,21 +168,6 @@ def get_failed_prereqs(test : dict, successful_calls : set, unsuccessful_calls :
 
     return failed_prereqs
 
-def get_reference_a_tag(api_call : str) -> str:
-    '''
-    Craft the link to the documentation page for a given API call.
-    
-    :param api_call: a string representing the API call, such as "directory/v1/users/list".
-    '''
-    if api_call == "groups-settings/v1/groups/get":
-        # The reference URL for this API is structured differently than the others
-        return '<a href="https://developers.google.com/admin-sdk/groups-settings/v1/reference/' \
-            f'groups/get">{api_call}</a>'
-    api = api_call.split('/')[0]
-    call = '/'.join(api_call.split('/')[1:])
-    return f'<a href="https://developers.google.com/admin-sdk/{api}/reference/rest/{call}">' \
-        f'{api_call}</a>'
-
 def get_failed_details(failed_prereqs : set) -> str:
     '''
     Create the string used for the Details column of the report when one
@@ -190,19 +176,9 @@ def get_failed_details(failed_prereqs : set) -> str:
     :param failed_prereqs: A set of strings with the API calls/function prerequisites
         that were not met for a given test.
     '''
-    api_links = {
-        api_call: get_reference_a_tag(api_call) for api_call in [
-            "directory/v1/users/list",
-            "directory/v1/orgunits/list",
-            "directory/v1/domains/list",
-            "directory/v1/groups/list",
-            "reports/v1/activities/list",
-            "groups-settings/v1/groups/get"
-        ]
-    }
 
-    failed_apis = [api_links[api] for api in failed_prereqs if api in api_links]
-    failed_functions = [call for call in failed_prereqs if call not in api_links]
+    failed_apis = [API_LINKS[api] for api in failed_prereqs if api in API_LINKS]
+    failed_functions = [call for call in failed_prereqs if call not in API_LINKS]
     failed_details = ""
     if len(failed_apis) > 0:
         links = ', '.join(failed_apis)

From 29f4fcd2918956d154b9e2dfe3579ebfa5dcce00 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 1 Feb 2024 15:02:56 -0800
Subject: [PATCH 33/35] Correct URL bugs

---
 scubagoggles/api_reference.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scubagoggles/api_reference.py b/scubagoggles/api_reference.py
index 63968441..50b0f455 100644
--- a/scubagoggles/api_reference.py
+++ b/scubagoggles/api_reference.py
@@ -16,8 +16,8 @@ class ApiUrl(Enum):
     LIST_DOMAINS = f"{BASE_URL}/directory/reference/rest/v1/domains/list"
     LIST_GROUPS = f"{BASE_URL}/directory/reference/rest/v1/groups/list"
     LIST_ACTIVITIES = f"{BASE_URL}/reports/reference/rest/v1/activities/list"
-    GET_GROUP = f"{BASE_URL}/groups-settings/v1/reference/"
+    GET_GROUP = f"{BASE_URL}/groups-settings/v1/reference/groups/get"
 
 API_LINKS = {
-    api.value: f'<a href="{ApiUrl[api.name]}">{api.value}</a>' for api in ApiReference
+    api.value: f'<a href="{ApiUrl[api.name].value}">{api.value}</a>' for api in ApiReference
 }
\ No newline at end of file

From 2cce5fcdbbdd6dcff21dbb4bed16865581ca9fb9 Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Thu, 1 Feb 2024 15:10:58 -0800
Subject: [PATCH 34/35] Add documentation

---
 scubagoggles/api_reference.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/scubagoggles/api_reference.py b/scubagoggles/api_reference.py
index 50b0f455..675805b6 100644
--- a/scubagoggles/api_reference.py
+++ b/scubagoggles/api_reference.py
@@ -1,8 +1,16 @@
+"""
+api_reference.py is where short-hand references and full URLs to the GWS api calls are maintained.
+
+"""
+
 from enum import Enum
 
 BASE_URL = "https://developers.google.com/admin-sdk"
 
 class ApiReference(Enum):
+    '''
+    Enum for mapping code-friendly names of the various API calls to their short-hand reference
+    '''
     LIST_USERS = "directory/v1/users/list"
     LIST_OUS = "directory/v1/orgunits/list"
     LIST_DOMAINS = "directory/v1/domains/list"
@@ -11,6 +19,9 @@ class ApiReference(Enum):
     GET_GROUP = "groups-settings/v1/groups/get"
 
 class ApiUrl(Enum):
+    '''
+    Enum for mapping code-friendly names of the various API calls to their documentation URLs 
+    '''
     LIST_USERS = f"{BASE_URL}/directory/reference/rest/v1/users/list"
     LIST_OUS = f"{BASE_URL}/directory/reference/rest/v1/orgunits/list"
     LIST_DOMAINS = f"{BASE_URL}/directory/reference/rest/v1/domains/list"
@@ -18,6 +29,7 @@ class ApiUrl(Enum):
     LIST_ACTIVITIES = f"{BASE_URL}/reports/reference/rest/v1/activities/list"
     GET_GROUP = f"{BASE_URL}/groups-settings/v1/reference/groups/get"
 
+# Dictionary mapping short-hand reference to <a> tags linking to the documentation
 API_LINKS = {
     api.value: f'<a href="{ApiUrl[api.name].value}">{api.value}</a>' for api in ApiReference
-}
\ No newline at end of file
+}

From 1273cab6a5f2498ff80bdcf81961b8b16f02cbfe Mon Sep 17 00:00:00 2001
From: Alden Hilton <adhilto@sandia.gov>
Date: Fri, 2 Feb 2024 10:17:19 -0800
Subject: [PATCH 35/35] Rename api_reference to types

---
 scubagoggles/provider.py                    | 2 +-
 scubagoggles/reporter/reporter.py           | 2 +-
 scubagoggles/{api_reference.py => types.py} | 0
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename scubagoggles/{api_reference.py => types.py} (100%)

diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py
index 5dd4e05f..60154140 100644
--- a/scubagoggles/provider.py
+++ b/scubagoggles/provider.py
@@ -6,7 +6,7 @@
 from tqdm import tqdm
 
 from scubagoggles.utils import create_subset_inverted_dict, create_key_to_list, merge_dicts
-from scubagoggles.api_reference import ApiReference
+from scubagoggles.types import ApiReference
 from scubagoggles.robust_dns import RobustDNSClient
 
 EVENTS = {
diff --git a/scubagoggles/reporter/reporter.py b/scubagoggles/reporter/reporter.py
index f5f83cd6..bf35a87d 100644
--- a/scubagoggles/reporter/reporter.py
+++ b/scubagoggles/reporter/reporter.py
@@ -9,7 +9,7 @@
 from datetime import datetime
 import pandas as pd
 from scubagoggles.utils import rel_abs_path
-from scubagoggles.api_reference import API_LINKS
+from scubagoggles.types import API_LINKS
 
 SCUBA_GITHUB_URL = "https://github.com/cisagov/scubagoggles"
 
diff --git a/scubagoggles/api_reference.py b/scubagoggles/types.py
similarity index 100%
rename from scubagoggles/api_reference.py
rename to scubagoggles/types.py