Enhance Error handling (#163)

* Implement warning for missing output * Add missing reo spf check * Add rego check for 10.2 * Add rego check for 15.2 * Add rego check for 16.2 * Add missing rego checks for group 18 * Broke up the run_reporter function to satisfy linter * Initial commit of error handling code * Merge latest changes from main * Fix bad merge * Simplify Tenant Metadata (#152) * Simplify tenant metadata section and correct groups bug * Remove scope that is no longer needed * Remove unneeded OAUTH scope from readme * Merge latest changes from #158, Ensure Each Control Is Included in the Reports * Fix bad merge * Correct gmail bug, forgot to preface the dns function calls with self * Refactor long lines * Broke out error handling code into separate functions * correct error with f string * Add special case prereqs to rego files * Modify groups prereqs * Correct typo in reports API reference * If prereqs not defined, assume dependence on reports api * Fix error in provider from merge * Add missing tests for gws 18 * Fix other regressions caused by merge * Make the linter happy * Refactor long lines * Fix groups API reference links * Refactor services object to a state variable * Make customer_id and dnsclient state variables * Ensure the domains() API is called only once * Changes to satisfy the linter * Convert API reference to enums * Correct URL bugs * Add documentation * Rename api_reference to types
cisagov · Feb 7, 2024 · 56b1890 · 56b1890
1 parent d6907a1
commit 56b1890
Show file tree

Hide file tree

Showing 13 changed files with 870 additions and 499 deletions.
diff --git a/Testing/RegoTests/gmail/gmail03_test.rego b/Testing/RegoTests/gmail/gmail03_test.rego
@@ -5,9 +5,31 @@ import future.keywords
 #
 # GWS.GMAIL.3.1v0.1
 #--
+test_MaintainList_Correct_V1 if {
+    # Test not implemented
+    PolicyId := "GWS.GMAIL.17.1v0.1"
+    Output := tests with input as {
+        "gmail_logs": {"items": [
+        ]},
+        "tenant_info": {
+            "topLevelOU": ""
+        }
+    }
+
+    RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId]
+    count(RuleOutput) == 1
+    not RuleOutput[0].RequirementMet
+    not RuleOutput[0].NoSuchEvent
+    RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
+}
+#--
+
+#
+# GWS.GMAIL.3.2v0.1
+#--
 test_SPF_Correct_V1 if {
     # Test SPF when there's only one domain
-    PolicyId := "GWS.GMAIL.3.1v0.1"
+    PolicyId := "GWS.GMAIL.3.2v0.1"
     Output := tests with input as {
         "spf_records": [
             {
@@ -27,7 +49,7 @@ test_SPF_Correct_V1 if {
 
 test_SPF_Correct_V2 if {
     # Test SPF when there's multiple domains
-    PolicyId := "GWS.GMAIL.3.1v0.1"
+    PolicyId := "GWS.GMAIL.3.2v0.1"
     Output := tests with input as {
         "spf_records": [
             {
@@ -51,7 +73,7 @@ test_SPF_Correct_V2 if {
 
 test_SPF_Incorrect_V1 if {
     # Test SPF when there's multiple domains and only one is correct
-    PolicyId := "GWS.GMAIL.3.1v0.1"
+    PolicyId := "GWS.GMAIL.3.2v0.1"
     Output := tests with input as {
         "spf_records": [
             {
@@ -75,7 +97,7 @@ test_SPF_Incorrect_V1 if {
 
 test_SPF_Incorrect_V2 if {
     # Test SPF when there's only one domain and it's wrong
-    PolicyId := "GWS.GMAIL.3.1v0.1"
+    PolicyId := "GWS.GMAIL.3.2v0.1"
     Output := tests with input as {
         "spf_records": [
             {

diff --git a/Testing/RegoTests/gmail/gmail10_test.rego b/Testing/RegoTests/gmail/gmail10_test.rego
@@ -310,5 +310,26 @@ test_GoogleWorkspaceSync_Incorrect_V5 if {
     not RuleOutput[0].NoSuchEvent
     RuleOutput[0].ReportDetails == "Requirement failed in Secondary OU."
 }
+#--
+
+#
+# GWS.GMAIL.10.2v0.1
+#--
+test_May_Correct_V1 if {
+    # Test not implemented
+    PolicyId := "GWS.GMAIL.10.2v0.1"
+    Output := tests with input as {
+        "gmail_logs": {"items": [
+        ]},
+        "tenant_info": {
+            "topLevelOU": ""
+        }
+    }
 
+    RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId]
+    count(RuleOutput) == 1
+    not RuleOutput[0].RequirementMet
+    not RuleOutput[0].NoSuchEvent
+    RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
+}
 #--
diff --git a/Testing/RegoTests/gmail/gmail15_test.rego b/Testing/RegoTests/gmail/gmail15_test.rego
@@ -348,4 +348,26 @@ test_EnhancedPreDeliveryMessageScanning_Incorrect_V5 if {
     not RuleOutput[0].NoSuchEvent
     RuleOutput[0].ReportDetails == "Requirement failed in Secondary OU."
 }
+#--
+
+#
+# GWS.GMAIL.15.2v0.1
+#--
+test_Other_Correct_V1 if {
+    # Test not implemented
+    PolicyId := "GWS.GMAIL.15.2v0.1"
+    Output := tests with input as {
+        "gmail_logs": {"items": [
+        ]},
+        "tenant_info": {
+            "topLevelOU": ""
+        }
+    }
+
+    RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId]
+    count(RuleOutput) == 1
+    not RuleOutput[0].RequirementMet
+    not RuleOutput[0].NoSuchEvent
+    RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
+}
 #--
diff --git a/Testing/RegoTests/gmail/gmail16_test.rego b/Testing/RegoTests/gmail/gmail16_test.rego
@@ -310,4 +310,26 @@ test_SecuritySandbox_Incorrect_V5 if {
     not RuleOutput[0].NoSuchEvent
     RuleOutput[0].ReportDetails == "Requirement failed in Secondary OU."
 }
+#--
+
+#
+# GWS.GMAIL.16.2v0.1
+#--
+test_Other_Correct_V1 if {
+    # Test not implemented
+    PolicyId := "GWS.GMAIL.16.2v0.1"
+    Output := tests with input as {
+        "gmail_logs": {"items": [
+        ]},
+        "tenant_info": {
+            "topLevelOU": ""
+        }
+    }
+
+    RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId]
+    count(RuleOutput) == 1
+    not RuleOutput[0].RequirementMet
+    not RuleOutput[0].NoSuchEvent
+    RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
+}
 #--
diff --git a/Testing/RegoTests/gmail/gmail18_test.rego b/Testing/RegoTests/gmail/gmail18_test.rego
@@ -22,4 +22,48 @@ test_AdvanvedEmailContentFitlering_Correct_V1 if {
     not RuleOutput[0].NoSuchEvent
     RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
 }
+#--
+
+#
+# GWS.GMAIL.18.2v0.1
+#--
+test_Other_Correct_V1 if {
+    # Test not implemented
+    PolicyId := "GWS.GMAIL.18.2v0.1"
+    Output := tests with input as {
+        "gmail_logs": {"items": [
+        ]},
+        "tenant_info": {
+            "topLevelOU": ""
+        }
+    }
+
+    RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId]
+    count(RuleOutput) == 1
+    not RuleOutput[0].RequirementMet
+    not RuleOutput[0].NoSuchEvent
+    RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
+}
+#--
+
+#
+# GWS.GMAIL.18.3v0.1
+#--
+test_PII_Correct_V1 if {
+    # Test not implemented
+    PolicyId := "GWS.GMAIL.18.3v0.1"
+    Output := tests with input as {
+        "gmail_logs": {"items": [
+        ]},
+        "tenant_info": {
+            "topLevelOU": ""
+        }
+    }
+
+    RuleOutput := [Result | some Result in Output; Result.PolicyId == PolicyId]
+    count(RuleOutput) == 1
+    not RuleOutput[0].RequirementMet
+    not RuleOutput[0].NoSuchEvent
+    RuleOutput[0].ReportDetails == "Currently not able to be tested automatically; please manually check."
+}
 #--
diff --git a/rego/Commoncontrols.rego b/rego/Commoncontrols.rego
@@ -713,6 +713,7 @@ tests contains {
 #--
 tests contains {
     "PolicyId": "GWS.COMMONCONTROLS.7.1v0.1",
+    "Prerequisites": ["directory/v1/users/list"],
     "Criticality": "Shall",
     "ReportDetails": concat("", [
         concat("", ["The following super admins are configured: ", concat(", ", SuperAdmins)]),

diff --git a/rego/Gmail.rego b/rego/Gmail.rego
@@ -85,6 +85,7 @@ DomainsWithDkim contains DkimRecord.domain if {
 
 tests contains {
     "PolicyId": "GWS.GMAIL.2.1v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "get_dkim_records"],
     "Criticality": "Should",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutDkim, AllDomains),
     "ActualValue": input.dkim_records,
@@ -105,14 +106,29 @@ if {
 #
 # Baseline GWS.GMAIL.3.1v0.1
 #--
+# No implementation steps provided for this policy
+tests contains {
+    "PolicyId": "GWS.GMAIL.3.1v0.1",
+    "Criticality": "Shall/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false
+}
+#--
+
+#
+# Baseline GWS.GMAIL.3.2v0.1
+#--
 DomainsWithSpf contains SpfRecord.domain if {
     some SpfRecord in input.spf_records
     some Rdata in SpfRecord.rdata
     startswith(Rdata, "v=spf1 ")
 }
 
 tests contains {
-    "PolicyId": "GWS.GMAIL.3.1v0.1",
+    "PolicyId": "GWS.GMAIL.3.2v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "get_spf_records"],
     "Criticality": "Shall",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutSpf, AllDomains),
     "ActualValue": DomainsWithoutSpf,
@@ -141,6 +157,7 @@ DomainsWithDmarc contains DmarcRecord.domain if {
 
 tests contains {
     "PolicyId": "GWS.GMAIL.4.1v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "get_dmarc_records"],
     "Criticality": "Shall",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutDmarc, AllDomains),
     "ActualValue": input.dmarc_records,
@@ -164,6 +181,7 @@ DomainsWithPreject contains DmarcRecord.domain if {
 
 tests contains {
     "PolicyId": "GWS.GMAIL.4.2v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "get_dmarc_records"],
     "Criticality": "Shall",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutPreject, AllDomains),
     "ActualValue": input.dmarc_records,
@@ -187,6 +205,7 @@ DomainsWithDHSContact contains DmarcRecord.domain if {
 
 tests contains {
     "PolicyId": "GWS.GMAIL.4.3v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "get_dmarc_records"],
     "Criticality": "Shall",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutDHSContact, AllDomains),
     "ActualValue": input.dmarc_records,
@@ -210,6 +229,7 @@ DomainsWithAgencyContact contains DmarcRecord.domain if {
 
 tests contains {
     "PolicyId": "GWS.GMAIL.4.4v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "get_dmarc_records"],
     "Criticality": "Should",
     "ReportDetails": ReportDetailsArray(Status, DomainsWithoutAgencyContact, AllDomains),
     "ActualValue": input.dmarc_records,
@@ -515,7 +535,8 @@ tests contains {
     "ReportDetails": "Currently not able to be tested automatically; please manually check.",
     "ActualValue": "",
     "RequirementMet": false,
-    "NoSuchEvent": false}
+    "NoSuchEvent": false
+}
 #--
 
 ###############
@@ -1347,6 +1368,19 @@ if {
 }
 #--
 
+#
+# Baseline GWS.GMAIL.10.2v0.1
+#--
+# No implementation steps provided for this policy
+tests contains {
+    "PolicyId": "GWS.GMAIL.10.2v0.1",
+    "Criticality": "May/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false
+}
+#--
 
 ################
 # GWS.GMAIL.11 #
@@ -1606,6 +1640,19 @@ if {
 }
 #--
 
+#
+# Baseline GWS.GMAIL.15.2v0.1
+#--
+# No implementation steps provided for this policy
+tests contains {
+    "PolicyId": "GWS.GMAIL.15.2v0.1",
+    "Criticality": "Should/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false
+}
+#--
 
 ################
 # GWS.GMAIL.16 #
@@ -1658,6 +1705,19 @@ if {
 }
 #--
 
+#
+# Baseline GWS.GMAIL.16.2v0.1
+#--
+# No implementation steps provided for this policy
+tests contains {
+    "PolicyId": "GWS.GMAIL.16.2v0.1",
+    "Criticality": "Should/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false
+}
+#--
 
 ################
 # GWS.GMAIL.17 #
@@ -1696,4 +1756,34 @@ tests contains {
     "RequirementMet": false,
     "NoSuchEvent": false
 }
+#--
+
+#
+# Baseline GWS.GMAIL.18.2v0.1
+#--
+# At this time we are unable to test because settings are configured in the GWS Admin Console
+# and not available within the generated logs
+tests contains {
+    "PolicyId": "GWS.GMAIL.18.2v0.1",
+    "Criticality": "Should/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false
+}
+#--
+
+#
+# Baseline GWS.GMAIL.18.3v0.1
+#--
+# At this time we are unable to test because settings are configured in the GWS Admin Console
+# and not available within the generated logs
+tests contains {
+    "PolicyId": "GWS.GMAIL.18.3v0.1",
+    "Criticality": "Shall/Not-Implemented",
+    "ReportDetails": "Currently not able to be tested automatically; please manually check.",
+    "ActualValue": "",
+    "RequirementMet": false,
+    "NoSuchEvent": false
+}
 #--
diff --git a/rego/Groups.rego b/rego/Groups.rego
@@ -356,6 +356,7 @@ NonCompliantGroups7_1 contains Group.name if {
 # if there are no groups, it has to be safe.
 tests contains {
     "PolicyId": "GWS.GROUPS.7.1v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "directory/v1/groups/list", "groups-settings/v1/groups/get"],
     "Criticality": "Should",
     "ReportDetails": NoGroupsDetails(Groups),
     "ActualValue": NoGroupsDetails(Groups),
@@ -370,6 +371,7 @@ if {
 # if there are groups
 tests contains {
     "PolicyId": "GWS.GROUPS.7.1v0.1",
+    "Prerequisites": ["directory/v1/domains/list", "directory/v1/groups/list", "groups-settings/v1/groups/get"],
     "Criticality": "Should",
     "ReportDetails": ReportDetailsGroups(NonCompliantGroups7_1),
     "ActualValue": {"NonCompliantGroups": NonCompliantGroups7_1},