From 915794895579a62561d34c0f2418b1f2f2bfb709 Mon Sep 17 00:00:00 2001 From: Roy Lane Date: Wed, 18 Dec 2024 15:51:52 -0500 Subject: [PATCH] policy_api: implement policy reduction and apply default values scuba_argument_parser: fix bug in converting argument value data types drive rego: fix 6.1 for subOUs/groups policy_api: add ability to dump Google's raw policy response installation & OPA instructions: add more detail about setup downloading OPA --- docs/installation/DownloadAndInstall.md | 15 +- docs/installation/OPA.md | 39 +- scubagoggles/policy_api.py | 648 +++++++++++++++++------- scubagoggles/provider.py | 4 +- scubagoggles/rego/Drive.rego | 14 +- scubagoggles/scuba_argument_parser.py | 7 +- 6 files changed, 513 insertions(+), 214 deletions(-) diff --git a/docs/installation/DownloadAndInstall.md b/docs/installation/DownloadAndInstall.md index f3f530a7..0726ffda 100644 --- a/docs/installation/DownloadAndInstall.md +++ b/docs/installation/DownloadAndInstall.md @@ -1,17 +1,19 @@ # Getting started > [!IMPORTANT] -> Use of this tool requires access to an internet browser for initial setup and to view the html report output. +> Use of this tool requires access to an internet browser for initial setup +> and to view the html report output. Setting up to run ScubaGoggles for the first time involves the following steps: 1. Install [Python 3](https://www.python.org/) on your system. 2. (Optional) Create and activate a Python virtual environment. 3. Install ScubaGoggles and dependencies into the Python environment. -4. Run ScubaGoggles setup to specify the output directory, the location of the - OPA executable, and the credentials file. -5. Download the Open Policy Agent (OPA) executable. -6. Create a Google OAuth credential file, unless you'll be using a Google +4. Run `scubagoggles setup` to specify the output directory, the location of the + OPA executable, and the credentials file. By default, the setup will + download the Open Policy Agent (OPA) + executable. +5. Create a Google OAuth credential file, unless you'll be using a Google service account. ## Install Python 3 @@ -127,7 +129,8 @@ location of the output directory. The ScubaGoggles setup utility lets you configure the data directory location, as well as the locations of the OPA executable and the Google credentials file. It is perfectly fine to locate the OPA executable and credentials files in the -output directory you create. +output directory you create. Unless you specify otherwise, the OPA executable +will be downloaded to the location you specify. When you run the setup utility, it will create a configuration file in your top-level user directory called `.scubagoggles` (**Note** the leading dot (.) diff --git a/docs/installation/OPA.md b/docs/installation/OPA.md index 762d0a3b..ed422e50 100644 --- a/docs/installation/OPA.md +++ b/docs/installation/OPA.md @@ -1,9 +1,16 @@ # Download the OPA executable -The tool makes use of [Open Policy Agent's Rego Policy language](https://www.openpolicyagent.org/docs/latest/policy-language/). -An OPA executable is required to execute this tool and can be downloaded -with the `scubagoggles getopa` command. +The tool makes use of [Open Policy Agent's Rego Policy language] +(https://www.openpolicyagent.org/docs/latest/policy-language/). By default, +the `scubagoggles setup` command downloads the OPA executable. You will only +need to download the OPA executable separately if you need a **specific** +version. Otherwise, you may skip this step and continue to +[Prerequisites](../prerequisites/Prerequisites.md). + +You may download the OPA executable, to either upgrade the version you +currently have or use a specific version, using the `scubagoggles getopa` +command: ``` scubagoggles getopa --help @@ -20,24 +27,34 @@ options: --opa_directory , -r Directory containing OPA executable (default: location established by setup) ``` -``` +```bash # example scubagoggles getopa -v v0.60.0 ``` If you have run the [ScubaGoggles setup utility](DownloadAndInstall.md#ScubaGoggles-Setup-Utility), you will have specified the location of the OPA executable. This location is -used by `getopa` when downloading the OPA executable. +used by `getopa` when downloading the OPA executable. Optionally, you may +download the executable to a location that is in the PATH environment variable. + +## Downloading the OPA Executable from the OPA Website -1. If the above script can not execute for any reason or you would prefer to download OPA manually, go to the [Open Policy Agent website](https://www.openpolicyagent.org/docs/latest/#running-opa) -2. Check the website for a compatible OPA version (Currently v0.45.0 and above) for ScubaGoggles and select the corresponding version on top left of the website -3. Navigate to the menu on left side of the screen: `Introduction -> Running OPA -> Download OPA` -4. Follow the instructions for downloading the respective OPA executable for your OS. +1. If the above script can not execute for any reason or you would prefer to + download OPA manually, go to the [Open Policy Agent website] + (https://www.openpolicyagent.org/docs/latest/#running-opa) +2. Check the website for a compatible OPA version (Currently v0.45.0 and above) + for ScubaGoggles and select the corresponding version on top left of the + website. +3. Navigate to the menu on left side of the screen: + `Introduction -> Running OPA -> Download OPA` +4. Follow the instructions for downloading the respective OPA executable for + your OS. > [!NOTE] > For linux and macOS, you must make sure the OPA executable has execute -> permission. If you downloaded the OPA executable using the `getopa` -> subcommand, the permission has already been set correctly. +> permission. If you downloaded the OPA executable either during the setup +> process or using the `getopa`subcommand, the permission has already been set +> correctly. ```bash # give the opa executable execute permissions diff --git a/scubagoggles/policy_api.py b/scubagoggles/policy_api.py index 1b3847ad..0494ffd6 100644 --- a/scubagoggles/policy_api.py +++ b/scubagoggles/policy_api.py @@ -1,10 +1,17 @@ """Google Policy API class implementation """ +import io +import json import logging +import os import re +import sys +import time from collections import defaultdict +from collections.abc import Iterable +from pathlib import Path from random import random from time import sleep @@ -50,61 +57,87 @@ class PolicyAPI: isDuration = lambda x: isinstance(x, str) and re.match(r'(?i)^\d+[hms]$', x) - # This is the complete list of policy settings returned by Google. - # The key is the setting type (NOTE that while Google returns a + # There may be duplicate policies returned for an orgunit/group and + # section. The policies must be "reduced" to single settings using + # a method. The default "reducer" method is to select the policy with the + # highest "sort order", and this is applied if a reducer is not specified + # in the expected policy settings below. + + _merge_reducer = '_merge' + + # This is the list of policy settings returned by Google that are relevant + # to the secure baselines. Settings that are not relevant are ignored + # (i.e., they are NOT included in this data structure - these are settings + # that are not referenced in any of our policy code). The key is the section + # (what Google calls "setting type") (NOTE that while Google returns a # setting type with a period (.) delimiting the type (e.g., # "sites.service_status"), we convert the periods into underscores because - # Rego uses the period to delimit hierarchy). The value is a dictionary - # containing the name of each setting and a function that will validate - # that the setting value is correct. - + # Rego uses the period to delimit hierarchy). The setting is a dictionary + # containing the name of each setting and a function that will validate that + # the setting value is correct. Optionally, a reducer may be given if it + # differs from the default ("maximum sort order") - this is set using the + # "reducer" key. + # + # The entries are ordered alphabetically (not that they have to, but it + # does make finding them in this data structure easier). If you need to + # add a new entry, it may be easiest to copy a similar entry and update + # it with the correct section/setting names and verifier, and finally + # whether a reducer is needed. _expectedPolicySettings = { - 'analytics_service_status': {'serviceState': isState}, - 'applied_digital_skills_service_status': {'serviceState': isState}, - 'appsheet_service_status': {'serviceState': isState}, - 'blogger_service_status': {'serviceState': isState}, - 'blogger_user_takeout': {'takeoutStatus': isEnum}, - 'books_user_takeout': {'takeoutStatus': isEnum}, - 'calendar_appointment_schedules': {'enablePayments': isBool}, - 'calendar_external_invitations': {'warnOnInvite': isBool}, - 'calendar_interoperability': { + 'analytics_service_status': {'settings': {'serviceState': isState}}, + 'applied_digital_skills_service_status': {'settings': { + 'serviceState': isState}}, + 'appsheet_service_status': {'settings': {'serviceState': isState}}, + 'blogger_service_status': {'settings': {'serviceState': isState}}, + 'blogger_user_takeout': {'settings': {'takeoutStatus': isEnum}}, + 'books_user_takeout': {'settings': {'takeoutStatus': isEnum}}, + 'calendar_appointment_schedules': {'settings': { + 'enablePayments': isBool}}, + 'calendar_external_invitations': {'settings': {'warnOnInvite': isBool}}, + 'calendar_interoperability': {'settings': { 'enableExchangeRoomBooking': isBool, 'enableFullEventDetails': isBool, - 'enableInteroperability': isBool}, - 'calendar_primary_calendar_max_allowed_external_sharing': { - 'maxAllowedExternalSharing': isEnum}, + 'enableInteroperability': isBool}}, + 'calendar_primary_calendar_max_allowed_external_sharing': {'settings': { + 'maxAllowedExternalSharing': isEnum}}, 'calendar_secondary_calendar_max_allowed_external_sharing': { - 'maxAllowedExternalSharing': isEnum}, - 'calendar_service_status': {'serviceState': isState}, - 'chat_chat_file_sharing': { + 'settings': {'maxAllowedExternalSharing': isEnum}}, + 'calendar_service_status': {'settings': {'serviceState': isState}}, + 'chat_chat_file_sharing': {'settings': { 'externalFileSharing': isEnum, - 'internalFileSharing': isEnum}, - 'chat_chat_history': { - 'allowUserModification': isBool, - 'historyOnByDefault': isBool}, - 'chat_external_chat_restriction': { + 'internalFileSharing': isEnum}}, + 'chat_chat_history': {'reducer': _merge_reducer, + 'settings': { + 'allowUserModification': isBool, + 'historyOnByDefault': isBool}}, + 'chat_external_chat_restriction': {'reducer': _merge_reducer, + 'settings': { 'allowExternalChat': isBool, - 'externalChatRestriction': isEnum}, - 'chat_service_status': {'serviceState': isState}, - 'chat_space_history': {'historyState': isEnum}, - 'chrome_canvas_service_status': {'serviceState': isState}, - 'classroom_api_data_access': {'enableApiAccess': isBool}, - 'classroom_class_membership': { + 'externalChatRestriction': isEnum}}, + 'chat_service_status': {'settings': {'serviceState': isState}}, + 'chat_space_history': {'settings': {'historyState': isEnum}}, + 'chrome_canvas_service_status': {'settings': {'serviceState': isState}}, + 'classroom_api_data_access': {'settings': {'enableApiAccess': isBool}}, + 'classroom_class_membership': {'settings': { 'whoCanJoinClasses': isEnum, - 'whichClassesCanUsersJoin': isEnum}, - 'classroom_guardian_access': { + 'whichClassesCanUsersJoin': isEnum}}, + 'classroom_guardian_access': {'settings': { 'allowAccess': isBool, - 'whoCanManageGuardianAccess': isEnum}, - 'classroom_roster_import': {'rosterImportOption': isEnum}, - 'classroom_student_unenrollment': {'whoCanUnenrollStudents': isEnum}, - 'classroom_teacher_permissions': {'whoCanCreateClasses': isEnum}, - 'cloud_search_service_status': {'serviceState': isState}, - 'drive_and_docs_drive_for_desktop': { + 'whoCanManageGuardianAccess': isEnum}}, + 'classroom_roster_import': {'settings': {'rosterImportOption': isEnum}}, + 'classroom_student_unenrollment': {'settings': { + 'whoCanUnenrollStudents': isEnum}}, + 'classroom_teacher_permissions': {'settings': { + 'whoCanCreateClasses': isEnum}}, + 'cloud_search_service_status': {'settings': {'serviceState': isState}}, + 'drive_and_docs_drive_for_desktop': {'settings': { 'allowDriveForDesktop': isBool, - 'restrictToAuthorizedDevices': isBool}, - 'drive_and_docs_drive_sdk': {'enableDriveSdkApiAccess': isBool}, - 'drive_and_docs_external_sharing': { + 'restrictToAuthorizedDevices': isBool}}, + 'drive_and_docs_drive_sdk': {'reducer': _merge_reducer, + 'settings': { + 'enableDriveSdkApiAccess': isBool}}, + 'drive_and_docs_external_sharing': {'settings': { 'accessCheckerSuggestions': isEnum, 'allowNonGoogleInvites': isBool, 'allowNonGoogleInvitesInAllowlistedDomains': isBool, @@ -114,42 +147,44 @@ class PolicyAPI: 'allowedPartiesForDistributingContent': isEnum, 'externalSharingMode': isEnum, 'warnForExternalSharing': isBool, - 'warnForSharingOutsideAllowlistedDomains': isBool}, - 'drive_and_docs_general_access_default': {'defaultFileAccess': isEnum}, - 'drive_and_docs_service_status': {'serviceState': isState}, - 'drive_and_docs_shared_drive_creation': { + 'warnForSharingOutsideAllowlistedDomains': isBool}}, + 'drive_and_docs_general_access_default': {'settings': { + 'defaultFileAccess': isEnum}}, + 'drive_and_docs_service_status': {'settings': { + 'serviceState': isState}}, + 'drive_and_docs_shared_drive_creation': {'settings': { 'allowContentManagersToShareFolders': isBool, 'allowExternalUserAccess': isBool, 'allowManagersToOverrideSettings': isBool, 'allowNonMemberAccess': isBool, 'allowSharedDriveCreation': isBool, 'allowedPartiesForDownloadPrintCopy': isEnum, - 'orgUnitForNewSharedDrives': isEnum}, - 'drive_and_docs_file_security_update': { + 'orgUnitForNewSharedDrives': isEnum}}, + 'drive_and_docs_file_security_update': {'settings': { 'allowUsersToManageUpdate': isBool, - 'securityUpdate': isEnum}, - 'enterprise_service_restrictions_service_status': { - 'serviceState': isState}, - 'gmail_auto_forwarding': {'enableAutoForwarding': isBool}, - 'gmail_email_attachment_safety': { + 'securityUpdate': isEnum}}, + 'enterprise_service_restrictions_service_status': {'settings': { + 'serviceState': isState}}, + 'gmail_auto_forwarding': {'settings': {'enableAutoForwarding': isBool}}, + 'gmail_email_attachment_safety': {'settings': { 'anomalousAttachmentProtectionConsequence': isEnum, 'applyFutureRecommendedSettingsAutomatically': isBool, 'attachmentWithScriptsProtectionConsequence': isEnum, 'enableAnomalousAttachmentProtection': isBool, 'enableAttachmentWithScriptsProtection': isBool, 'enableEncryptedAttachmentProtection': isBool, - 'encryptedAttachmentProtectionConsequence': isEnum}, - 'gmail_email_spam_filter_ip_allowlist': { - 'allowedIpAddresses': isListStrings}, - 'gmail_enhanced_pre_delivery_message_scanning': { - 'enableImprovedSuspiciousContentDetection': isBool}, - 'gmail_links_and_external_images': { + 'encryptedAttachmentProtectionConsequence': isEnum}}, + 'gmail_email_spam_filter_ip_allowlist': {'settings': { + 'allowedIpAddresses': isListStrings}}, + 'gmail_enhanced_pre_delivery_message_scanning': {'settings': { + 'enableImprovedSuspiciousContentDetection': isBool}}, + 'gmail_links_and_external_images': {'settings': { 'applyFutureSettingsAutomatically': isBool, 'enableAggressiveWarningsOnUntrustedLinks': isBool, 'enableExternalImageScanning': isBool, - 'enableShortenerScanning': isBool}, - 'gmail_service_status': {'serviceState': isState}, - 'gmail_spoofing_and_authentication': { + 'enableShortenerScanning': isBool}}, + 'gmail_service_status': {'settings': {'serviceState': isState}}, + 'gmail_spoofing_and_authentication': {'settings': { 'applyFutureSettingsAutomatically': isBool, 'detectDomainNameSpoofing': isBool, 'detectDomainSpoofingFromUnauthenticatedSenders': isBool, @@ -160,78 +195,151 @@ class PolicyAPI: 'domainSpoofingConsequence': isEnum, 'employeeNameSpoofingConsequence': isEnum, 'groupsSpoofingConsequence': isEnum, - 'unauthenticatedEmailConsequence': isEnum}, - 'groups_for_business_groups_sharing': { + 'unauthenticatedEmailConsequence': isEnum}}, + 'groups_for_business_groups_sharing': {'reducer': _merge_reducer, + 'settings': { 'collaborationCapability': isEnum, 'createGroupsAccessLevel': isEnum, 'newGroupsAreHidden': isBool, 'ownersCanAllowExternalMembers': isBool, 'ownersCanAllowIncomingMailFromPublic': isBool, 'ownersCanHideGroups': isBool, - 'viewTopicsDefaultAccessLevel': isEnum}, - 'groups_for_business_service_status': {'serviceState': isState}, - 'jamboard_service_status': {'serviceState': isState}, - 'keep_service_status': {'serviceState': isState}, - 'location_history_user_takeout': {'takeoutStatus': isEnum}, - 'maps_user_takeout': {'takeoutStatus': isEnum}, - 'meet_safety_access': {'meetingsAllowedToJoin': isEnum}, - 'meet_safety_domain': {'usersAllowedToJoin': isEnum}, - 'meet_safety_external_participants': {'enableExternalLabel': isBool}, - 'meet_safety_host_management': {'enableHostManagement': isBool}, - 'meet_service_status': {'serviceState': isState}, - 'meet_video_recording': {'enableRecording': isBool}, - 'migrate_service_status': {'serviceState': isState}, - 'pay_user_takeout': {'takeoutStatus': isEnum}, - 'photos_user_takeout': {'takeoutStatus': isEnum}, - 'play_console_user_takeout': {'takeoutStatus': isEnum}, - 'play_user_takeout': {'takeoutStatus': isEnum}, - 'security_advanced_protection_program': { + 'viewTopicsDefaultAccessLevel': isEnum}}, + 'groups_for_business_service_status': {'settings': { + 'serviceState': isState}}, + 'jamboard_service_status': {'settings': {'serviceState': isState}}, + 'keep_service_status': {'settings': {'serviceState': isState}}, + 'location_history_user_takeout': {'settings': { + 'takeoutStatus': isEnum}}, + 'maps_user_takeout': {'settings': {'takeoutStatus': isEnum}}, + 'meet_safety_access': {'settings': {'meetingsAllowedToJoin': isEnum}}, + 'meet_safety_domain': {'settings': {'usersAllowedToJoin': isEnum}}, + 'meet_safety_external_participants': {'settings': { + 'enableExternalLabel': isBool}}, + 'meet_safety_host_management': {'settings': { + 'enableHostManagement': isBool}}, + 'meet_service_status': {'settings': {'serviceState': isState}}, + 'meet_video_recording': {'settings': {'enableRecording': isBool}}, + 'migrate_service_status': {'settings': {'serviceState': isState}}, + 'pay_user_takeout': {'settings': {'takeoutStatus': isEnum}}, + 'photos_user_takeout': {'settings': {'takeoutStatus': isEnum}}, + 'play_console_user_takeout': {'settings': {'takeoutStatus': isEnum}}, + 'play_user_takeout': {'settings': {'takeoutStatus': isEnum}}, + 'security_advanced_protection_program': {'settings': { 'enableAdvancedProtectionSelfEnrollment': isBool, - 'securityCodeOption': isEnum}, - 'security_less_secure_apps': {'allowLessSecureApps': isBool}, - 'security_login_challenges': {'enableEmployeeIdChallenge': isBool}, - 'security_password': { + 'securityCodeOption': isEnum}}, + 'security_less_secure_apps': {'reducer': _merge_reducer, + 'settings': { + 'allowLessSecureApps': isBool}}, + 'security_login_challenges': {'settings': { + 'enableEmployeeIdChallenge': isBool}}, + 'security_password': {'settings': { 'allowedStrength': isEnum, 'allowReuse': isBool, 'enforceRequirementsAtLogin': isBool, 'expirationDuration': isDuration, 'maximumLength': isInt, - 'minimumLength': isInt}, - 'security_session_controls': {'webSessionDuration': isDuration}, - 'security_super_admin_account_recovery': { - 'enableAccountRecovery': isBool}, - 'security_user_account_recovery': {'enableAccountRecovery': isBool}, - 'sites_service_status': {'serviceState': isState}, - 'sites_sites_creation_and_modification': { + 'minimumLength': isInt}}, + 'security_session_controls': {'settings': { + 'webSessionDuration': isDuration}}, + 'security_super_admin_account_recovery': {'reducer': _merge_reducer, + 'settings': { + 'enableAccountRecovery': isBool}}, + 'security_user_account_recovery': {'reducer': _merge_reducer, + 'settings': { + 'enableAccountRecovery': isBool}}, + 'sites_service_status': {'settings': {'serviceState': isState}}, + 'sites_sites_creation_and_modification': {'settings': { 'allowSitesCreation': isBool, - 'allowSitesModification': isBool}, - 'tasks_service_status': {'serviceState': isState}, - 'vault_service_status': {'serviceState': isState}, - 'workspace_marketplace_apps_access_options': { + 'allowSitesModification': isBool}}, + 'takeout_service_status': {'settings': {'serviceState': isState}}, + 'tasks_service_status': {'settings': {'serviceState': isState}}, + 'vault_service_status': {'settings': {'serviceState': isState}}, + 'workspace_marketplace_apps_access_options': {'settings': { 'accessLevel': isEnum, - 'allowAllInternalApps': isBool}, - 'youtube_user_takeout': {'takeoutStatus': isEnum}} + 'allowAllInternalApps': isBool}}, + 'youtube_user_takeout': {'settings': {'takeoutStatus': isEnum}}} + + # In this section of Google's Policy API documentation: + # https://cloud.google.com/identity/docs/concepts/policy-api-concepts + # #default_field_values + # there are a number of settings which may not be present in the top-level + # orgunit, and for these cases defaults must be applied. The following + # default values are taken directly from the documentation. Google claims + # that it would take an act of god to change these defaults, but since + # these are copied from an external source, there is a risk that one or + # more defaults may be changed. There is also no assurance that Google's + # documentation reflects the current default values (i.e., the + # documented defaults must be kept current with the implementation). + + _defaults = { + 'chat_chat_history': {'allowUserModification': True, + 'historyOnByDefault': False}, + 'chat_external_chat_restriction': { + 'allowExternalChat': False, + 'externalChatRestriction': 'NO_RESTRICTION'}, + 'drive_and_docs_drive_sdk': {'enableDriveSdkApiAccess': True}, + 'drive_and_docs_external_sharing': { + 'accessCheckerSuggestions': 'RECIPIENTS_OR_AUDIENCE_OR_PUBLIC', + 'allowNonGoogleInvites': True, + 'allowNonGoogleInvitesInAllowlistedDomains': False, + 'allowPublishingFiles': True, + 'allowReceivingExternalFiles': True, + 'allowReceivingFilesOutsideAllowlistedDomains': True, + 'allowedPartiesForDistributingContent': 'ALL_ELIGIBLE_USERS', + 'externalSharingMode': 'ALLOWED', + 'warnForExternalSharing': True, + 'warnForSharingOutsideAllowlistedDomains': True}, + 'drive_and_docs_general_access_default': { + 'defaultFileAccess': 'LINK_SHARING_PRIVATE'}, + 'gmail_workspace_sync_for_outlook': { + 'enableGoogleWorkspaceSyncForMicrosoftOutlook': True}, + 'gmail_email_spam_filter_ip_allowlist': { + 'allowedIpAddresses': []}, + 'groups_for_business_groups_sharing': { + 'collaborationCapability': 'DOMAIN_USERS_ONLY', + 'createGroupsAccessLevel': 'USERS_IN_DOMAIN', + 'newGroupsAreHidden': False, + 'ownersCanAllowExternalMembers': False, + 'ownersCanAllowIncomingMailFromPublic': True, + 'ownersCanHideGroups': False, + 'viewTopicsDefaultAccessLevel': 'DOMAIN_USERS'}, + 'security_less_secure_apps': {'allowLessSecureApps': False}, + 'security_super_admin_account_recovery': { + 'enableAccountRecovery': False}, + 'security_user_account_recovery': {'enableAccountRecovery': False}, + 'workspace_marketplace_apps_access_options': { + 'accessLevel': 'ALLOW_ALL', + 'allowAllInternalApps': False}, + } # This is the URL to the Policies API. _baseURL = 'https://cloudidentity.googleapis.com/v1beta1/policies' _too_many_requests = 429 - def __init__(self, gws_auth: GwsAuth): + def __init__(self, gws_auth: GwsAuth, top_orgunit: str): - """PolicyAPI class instance initialization + """PolicyAPI class instance initialization. :param GwsAuth gws_auth: GWS credentials instance. + :param str top_orgunit: name of the top-level orgunit. """ # Google's AuthorizedSession is currently being used because this # API is not available in the Google API Client interface. self._session = AuthorizedSession(gws_auth.credentials) + self._top_orgunit = top_orgunit + # This is a mapping of Google's org unit ids to names. self._orgunit_id_map = self._get_ou() self._group_id_map = self._get_groups() + # This is a dictionary that is used in reducing the policies returned + # by Google. + self._reduction_map = None + def __enter__(self): # This class is implemented as a context manager - meaning that it's @@ -267,37 +375,24 @@ def get_policies(self) -> dict: # contains one or more values and is associated with an org unit. policies = self._get_policies_list() - result = defaultdict(dict) + # If the following environment variable is defined, the policy data + # returned by Google, as well as the orgunit and group maps, will be + # written to the file name defined in the variable. If the value + # contains only whitespace, the data will be written to the standard + # output stream. This is intended for debugging. + dump_envname = 'SCUBAGOGGLES_DUMP_FILE' + if dump_envname in os.environ: + self._dump(policies, os.environ[dump_envname].strip()) - for policy in policies: + self._reduce(policies) - # For the current policy setting, use the returned org unit id - # to get the org unit name, which is used as the key for the - # result dictionary. - orgunit_id = policy['policyQuery']['orgUnit'] - orgunit_id = orgunit_id.removeprefix('orgUnits/') - orgunit_name = self._orgunit_id_map[orgunit_id]['name'] + result = defaultdict(dict) - if 'group' in policy['policyQuery']: - group_id = policy['policyQuery']['group'] - group_id = group_id.removeprefix('groups/') - group_name = self._group_id_map[group_id] - orgunit_name += f' (group "{group_name}")' + for key, policy in self._reduction_map.items(): + orgunit_name, section = key + result[orgunit_name][section] = self._settings(policy) - # The setting has two layers in the policies dictionary. Depending - # on the setting, there may be one or multiple values - so the - # setting itself has a "type" (i.e., "name), and the value is a - # dictionary with one or more name/value pairs. The setting type - # is something like "settings/appsheet.service_status". We remove - # the "settings/" prefix and convert the dot to an underscore. - # This results in "appsheet_service_status", which is a format that - # can easily be used in Rego code. - policy_setting = policy['setting'] - setting_type = policy_setting['type'].removeprefix('settings/') - setting_type = setting_type.replace('.', '_') - setting_value = policy_setting['value'] - - result[orgunit_name][setting_type] = setting_value + self._apply_defaults(result) return result @@ -418,21 +513,17 @@ def _get(self, url: str, params: dict = None) -> dict: response = None - # Google will return the "too many requests" error if the requests - # come in without any delay between them. Is there a better way than - # having to delay (like telling Google up front how many requests - # we'll be making in a row, or is it a "quota" that needs to be - # adjusted??)? Anyway, the following loop implements the delay. The - # total iterations is limited to 8 because the delay is exponential - # and after several iterations the delay becomes impractical if the - # error continues to be returned. - # - # Subsequent note: this may not be necessary because we're now making - # only 1 (with possible next page requests) call, and it's doubtful - # that this would trigger a "too many requests" response. However, - # it's here in case it's needed. - - for iter_count in range(8): + start_time = time.time() + + # Google will return the "too many requests" error if the requests come + # in without any delay between them. The total iterations is limited to + # 8 because the delay is exponential and after several iterations the + # delay becomes impractical if the error continues to be returned. + # In practice, with the API calls it sometimes takes seconds to get a + # response, and for those cases a subsequent request is made after a + # sufficient delay due to the time it took for the previous response. + + for iter_count in range(1, 9): response = self._session.get(url, params = params) if response.status_code != self._too_many_requests: @@ -445,82 +536,267 @@ def _get(self, url: str, params: dict = None) -> dict: delay += (delay * random()) log.debug('attempt %i - too many requests response: ' - 'delay %i seconds', iter_count + 1, delay) + 'delay %.2f seconds', iter_count, delay) sleep(delay) + end_time = time.time() - start_time + response_json = response.json() if not response.ok: raise RuntimeError(f'? {url} - {response_json["error"]["message"]}') - return response.json() + log.debug('URL: %s', url) + if params: + log.debug(' params: %s', params) + log.debug('Result length: %d', len(response.text)) + log.debug('Elapsed time: %.2f seconds', end_time) + + return response_json + + def _reduce(self, policies: Iterable): + + """Reduces the policies returned by Google to those that apply for + each orgunit and group. + + This method populates the instance's "_reduction_map". It is a + dictionary where the key is a tuple containing the orgunit/group name + and section name. The value is a policy (dictionary). After the + reduction, there will be only one policy for each section in an + orgunit/group. + + See https://cloud.google.com/identity/docs/concepts/policy-api-concepts + #reducers_for_settings + for Google's discussion of the "reduction process". + + :param list policies: list of policies (dictionaries). + """ + + self._reduction_map = {} + + # Sorting the policies by largest sort order first is KEY to getting + # the correct policies. This accomplishes the "max" reduction + # referred to by Google, and it also necessary for the "merge" + # reduction. + policies.sort(key = self._sort_order, reverse = True) + + for policy in policies: + + # For the current policy setting, use the returned org unit id + # to get the org unit name, which is used as the key for the + # result dictionary. + orgunit_id = policy['policyQuery']['orgUnit'] + orgunit_id = orgunit_id.removeprefix('orgUnits/') + orgunit_name = self._orgunit_id_map[orgunit_id]['name'] + + if 'group' in policy['policyQuery']: + group_id = policy['policyQuery']['group'] + group_id = group_id.removeprefix('groups/') + group_name = self._group_id_map[group_id] + orgunit_name += f' (group "{group_name}")' + + # The setting has two layers in the policies dictionary. Depending + # on the setting, there may be one or multiple values - so the + # setting itself has a "type" (i.e., "name), and the value is a + # dictionary with one or more name/value pairs. The setting type + # is something like "settings/appsheet.service_status". We remove + # the "settings/" prefix and convert the dot to an underscore. + # This results in "appsheet_service_status", which is a format that + # can easily be used in Rego code. + policy_setting = policy['setting'] + section = policy_setting['type'].removeprefix('settings/') + section = section.replace('.', '_') + + key = (orgunit_name, section) + + # This is where the dictionary is populated, with policies + # having the highest sort order for each orgunit/group and + # section. This is possible because of the initial sorting + # above. + if key not in self._reduction_map: + self._reduction_map[key] = policy + continue + + expected_settings = self._expectedPolicySettings.get(section) + + if not expected_settings: + # The section is not in the expected settings, which means + # it is not associated with a secure baseline. + continue + + # If there is a reducer associated with this policy, it will + # be invoked. + reduce_name = expected_settings.get('reduce_method') + + if reduce_name: + reduce_method = getattr(self, reduce_name) + reduce_method(key, policy) + + @staticmethod + def _settings(policy: dict) -> dict: + + """Given a policy (dictionary) from Google's response, returns the + name/value pairs, which are the setting values for a section. + + :param policy: "raw" policy data dictionary returned by Google. + """ + + return policy['setting']['value'] + + @staticmethod + def _sort_order(policy: dict) -> float: + + """Given a policy (dictionary) from Google's response, returns the + sort order value for the policy. + + :param policy: "raw" policy data dictionary returned by Google. + """ - def verify(self, orgunit, policies): + return policy['policyQuery']['sortOrder'] + + def _merge(self, key: tuple, policy: dict): + + """Peforms a merge reduction of the given policy with the current + policy (having the greatest sort order). + + :param key: tuple containing the orgunit name and section name, used + for locating the corresponding policy in the reduction map. + :param policy: "raw" policy having a sort order below the policy + stored in the reduction map. + """ + + # Get the policy from the reduction map. The policies have already + # been sorted by largest sort order first, so this one is the + # in-effect policy for the given key. + + current_policy = self._reduction_map[key] + + current_settings = self._settings(current_policy) + + # If any setting present in the given lower sort order policy is + # not present in the in-effect policy, add the setting to the + # current policy. + + for setting, value in self._settings(policy).items(): + if setting not in current_settings: + current_settings[setting] = value + + def _apply_defaults(self, policies: dict): + + """Applies Google's default setting values to the top orgunit policies. + See the discussion above where the default setting values are defined. + + :param policies: the complete set of policies, formatted from the raw + data returned by Google. + """ + + if not self._top_orgunit: + log.debug('No top orgunit specified in PolicyAPI - ' + 'skipping defaults') + return + + # The defaults apply only to the top-level orgunit. The top orgunit + # must contain all settings, and the subordinate orgunits and groups + # only contain settings that have changed from the top orgunit's + # values. We'll keep track of the default settings actually applied + # so they can be reported in the log. + + top_ou_policies = policies[self._top_orgunit] + + applied = defaultdict(dict) + + for section, settings in self._defaults.items(): + if section not in top_ou_policies: + # Copy the section dictionary into the top OU policies as a + # precaution because the defaults are read-only. + top_ou_policies[section] = settings.copy() + applied[section] = settings + continue + + for setting, value in settings.items(): + if setting not in top_ou_policies[section]: + top_ou_policies[section][setting] = value + applied[section][setting] = value + + if applied: + log.debug('Default value(s) applied to %s:', self._top_orgunit) + for section, settings in applied.items(): + log.debug(' %s:', section) + for setting, value in settings.items(): + log.debug(' %s: %s', setting, str(value)) + + def _dump(self, policies: Iterable, file_or_stream = sys.stdout): + + """Writes the orgunit and group maps, and the given policies from + Google to the given I/O stream or file. + + :param policies: sequence of policy data (dictionaries) returned by + Google's Policy API. + :param file_or_stream: [optional] file specification for the file + to be written, or an existing I/O stream. If not given or if + an empty string is provided, the standard output stream (stdout) + is used. + """ + + out_data = {'orgunits': self._orgunit_id_map, + 'groups': self._group_id_map, + 'policies': policies} + + is_stream = not file_or_stream or isinstance(file_or_stream, io.IOBase) + + # pylint: disable=consider-using-with + out_stream = (sys.stdout if not file_or_stream + else file_or_stream if is_stream + else Path(file_or_stream).open('wt', encoding = 'utf-8')) + + try: + json.dump(out_data, out_stream, indent = 2) + finally: + if not is_stream: + out_stream.close() + + def verify(self, policies: dict) -> bool: """Verify that all expected policy settings (see above) are present - for the given orgunit (i.e., the top orgunit), and that the values of - each setting are the correct type and/or format. + for the top-level orgunit, and that the values of each setting are the + correct type and/or format. We do this verification because while Rego is good at checking for policy requirements, it may yield incorrect results when expected - settings are missing or values are incorrect. This verification - only issues warnings. so we're not aborting if something is found - to be missing or incorrect. However, if warnings are issued, - checks should be done to determine what's wrong with the data returned - by Google. + settings are missing or values are incorrect. This verification only + issues warnings. so we're not aborting if something is found to be + missing or incorrect. However, if warnings are issued, checks should be + done to determine what's wrong with the data returned by Google. - :param str orgunit: name of the top-level orgunit. :param dict policies: policy settings returned by get_policies(). - :return: True if all expected policy settings are found and the - setting values are correct types and format. + :return: True if all expected policy settings are found and the setting + values are correct types and format. """ + orgunit = self._top_orgunit policies_ok = True - expected_settings = self._expectedPolicySettings + expected_policy_settings = self._expectedPolicySettings orgunit_policies = policies.get(orgunit) if not orgunit_policies: log.warning('No policy settings found for orgunit: %s', orgunit) return False - missing_settings = {n for n in expected_settings + missing_settings = {n for n in expected_policy_settings if n not in orgunit_policies} - # Temporary section: Google's API had an issue in the alpha version - # where all the service status settings were missing. These are - # used (necessary) in the Rego code, so as a workaround, the - # service status is assumed to be "enabled" if it is missing (except - # for sites, which has a baseline requirment that it be disabled). - # Although this code won't do anything once this bug is fixed by - # Google, this section should probably be removed when no longer - # needed. - - missing_service_status = {n for n in missing_settings - if n.endswith('_service_status') - and not n.startswith('sites_')} - - if missing_service_status: - for key in missing_service_status: - orgunit_policies[key] = {'serviceState': 'ENABLED'} - - log.warning('Service status setting(s) missing - ' - 'assumed enabled: %s', - missing_service_status) - missing_settings -= missing_service_status - - # End temporary section - if missing_settings: log.warning('Setting(s) missing from %s orgunit: %s', orgunit, str(sorted(missing_settings))) policies_ok = False - for resource_name, expected_settings in expected_settings.items(): + for section, section_data in expected_policy_settings.items(): - settings = orgunit_policies.get(resource_name) + expected_settings = section_data['settings'] + settings = orgunit_policies.get(section) if not settings: continue @@ -535,7 +811,7 @@ def verify(self, orgunit, policies): log.warning('Settings missing or values invalid for ' 'orgunit %s, resource %s: %s', orgunit, - resource_name, + section, sorted(invalid_settings)) return policies_ok diff --git a/scubagoggles/provider.py b/scubagoggles/provider.py index a3d29fef..7d0c4c54 100644 --- a/scubagoggles/provider.py +++ b/scubagoggles/provider.py @@ -599,9 +599,9 @@ def call_gws_providers(self, products: list, quiet) -> dict: f'outputs will be incorrect: {exc}', RuntimeWarning) self._unsuccessful_calls.add(ApiReference.LIST_ACTIVITIES.value) - with PolicyAPI(self._gws_auth) as policy_api: + with PolicyAPI(self._gws_auth, self._top_ou) as policy_api: policies = policy_api.get_policies() - policy_api.verify(self._top_ou, policies) + policy_api.verify(policies) product_to_items['policies'] = policies diff --git a/scubagoggles/rego/Drive.rego b/scubagoggles/rego/Drive.rego index 0ae397d9..0ecf7955 100644 --- a/scubagoggles/rego/Drive.rego +++ b/scubagoggles/rego/Drive.rego @@ -1651,14 +1651,16 @@ NonCompliantOUs6_1 contains { if { some OU, settings in input.policies DriveEnabled(OU) - desktopEnabled := utils.GetApiSettingValue("drive_and_docs_drive_for_desktop", - "allowDriveForDesktop", - OU) - allowAuthorized := utils.GetApiSettingValue("drive_and_docs_drive_for_desktop", - "restrictToAuthorizedDevices", - OU) + section := "drive_and_docs_drive_for_desktop" + desktopSetting := "allowDriveForDesktop" + desktopEnabled := utils.GetApiSettingValue(section, desktopSetting, OU) desktopEnabled + desktopSet := utils.ApiSettingExists(section, desktopSetting, OU) + authDevicesSetting := "restrictToAuthorizedDevices" + allowAuthorized := utils.GetApiSettingValue(section, authDevicesSetting, OU) not allowAuthorized + authDevicesSet := utils.ApiSettingExists(section, authDevicesSetting, OU) + true in {desktopSet, authDevicesSet} } tests contains { diff --git a/scubagoggles/scuba_argument_parser.py b/scubagoggles/scuba_argument_parser.py index 872fb36a..0c4d139d 100644 --- a/scubagoggles/scuba_argument_parser.py +++ b/scubagoggles/scuba_argument_parser.py @@ -148,9 +148,10 @@ def validate_config(args : argparse.Namespace) -> None: 'regopath') for option_name in path_value_options: - if (option_name in args - and not isinstance(getattr(args, option_name), Path)): - setattr(args, option_name, path_parser(args.credentials)) + if option_name in args: + option_value = getattr(args, option_name) + if not isinstance(option_value, Path): + setattr(args, option_name, path_parser(option_value)) if 'omitpolicy' in args: ScubaArgumentParser.validate_omissions(args)