From 02ec5d7d6bdbbef875db963f325f0e8a2fa6f72f Mon Sep 17 00:00:00 2001 From: Michael McMurray Date: Fri, 13 Jan 2023 12:32:29 -0500 Subject: [PATCH 1/7] aws - subnet - ip application filter --- c7n/resources/vpc.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/c7n/resources/vpc.py b/c7n/resources/vpc.py index 39c550ac165..2f44711ce92 100644 --- a/c7n/resources/vpc.py +++ b/c7n/resources/vpc.py @@ -2885,3 +2885,64 @@ def process(self, resources, event=None): results.append(resource) return results + + +@Subnet.filter_registry.register('ip-allocation-threshold') +class SubnetIpAllocationFilter(Filter): + """Filters subnets based on ip allocation percentage + + :example: + + .. code-block:: yaml + + policies: + - name: subnet-ip-threshold-policy + resource: subnet + filters: + - type: ip-allocation-threshold + percentage: 80 + op: gte + """ + schema = type_schema( + 'ip-allocation-threshold', + percentage={'type': 'number'}, + op={'enum': ['eq', 'ne', 'lt', 'gt', 'lte', 'gte']} + ) + + permissions = ("ec2:DescribeSubnets",) + + def calculate_ip_allocation(self, subnet): + subnetMask = subnet.get('CidrBlock').split('/')[1] + hostBits = 32 - int(subnetMask) + totalHost = ((2 ** hostBits) - 2) + availableHost = subnet.get('AvailableIpAddressCount') + ipsUsed = totalHost - availableHost + percentageOfIpsUsed = (ipsUsed / totalHost) * 100 + return percentageOfIpsUsed + + def process(self, resources, event=None): + results = [] + threshold_percentage = self.data.get('percentage') + op = self.data.get('op') + for subnet in resources: + percentage_used = self.calculate_ip_allocation(subnet) + match op: + case 'eq': + if threshold_percentage == percentage_used: + results.append(subnet) + case 'ne': + if threshold_percentage != percentage_used: + results.append(subnet) + case 'lt': + if percentage_used < threshold_percentage: + results.append(subnet) + case 'gt': + if percentage_used > threshold_percentage: + results.append(subnet) + case 'lte': + if (percentage_used < threshold_percentage) or (percentage_used == threshold_percentage): + results.append(subnet) + case 'gte': + if (percentage_used > threshold_percentage) or (percentage_used == threshold_percentage): + results.append(subnet) + return results From 2c167fad52c64f52721762ab5ff94d6a84209793 Mon Sep 17 00:00:00 2001 From: Gianncarlo Giannattasio <89796775+CarloGiannattasio@users.noreply.github.com> Date: Mon, 23 Jan 2023 11:15:52 -0500 Subject: [PATCH 2/7] vpc.py - ip-allocation-threshold - updated to work with python 3.7 --- c7n/resources/vpc.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/c7n/resources/vpc.py b/c7n/resources/vpc.py index 2f44711ce92..aabb7d12319 100644 --- a/c7n/resources/vpc.py +++ b/c7n/resources/vpc.py @@ -2926,23 +2926,22 @@ def process(self, resources, event=None): op = self.data.get('op') for subnet in resources: percentage_used = self.calculate_ip_allocation(subnet) - match op: - case 'eq': - if threshold_percentage == percentage_used: - results.append(subnet) - case 'ne': - if threshold_percentage != percentage_used: - results.append(subnet) - case 'lt': - if percentage_used < threshold_percentage: - results.append(subnet) - case 'gt': - if percentage_used > threshold_percentage: - results.append(subnet) - case 'lte': - if (percentage_used < threshold_percentage) or (percentage_used == threshold_percentage): - results.append(subnet) - case 'gte': - if (percentage_used > threshold_percentage) or (percentage_used == threshold_percentage): - results.append(subnet) + if op == 'eq': + if threshold_percentage == percentage_used: + results.append(subnet) + elif op == 'ne': + if threshold_percentage != percentage_used: + results.append(subnet) + elif op == 'lt': + if percentage_used < threshold_percentage: + results.append(subnet) + elif op == 'gt': + if percentage_used > threshold_percentage: + results.append(subnet) + elif op == 'lte': + if (percentage_used < threshold_percentage) or (percentage_used == threshold_percentage): + results.append(subnet) + elif op == 'gte': + if (percentage_used > threshold_percentage) or (percentage_used == threshold_percentage): + results.append(subnet) return results From 15e1286b7ffd24b17fdf9c728a552e48368ca33d Mon Sep 17 00:00:00 2001 From: Gianncarlo Giannattasio Date: Fri, 20 Oct 2023 10:22:13 -0400 Subject: [PATCH 3/7] update API logic --- Pipfile | 84 +++++++++++++++++ c7n/manager.py | 16 +++- c7n/resource_metadata_update_with_email.py | 102 +++++++++++++++++++++ c7n/resources/asg.py | 3 +- c7n/resources/rds.py | 3 +- c7n/resources/vpc.py | 60 ------------ tests/conftest.py | 2 +- 7 files changed, 204 insertions(+), 66 deletions(-) create mode 100644 Pipfile create mode 100644 c7n/resource_metadata_update_with_email.py diff --git a/Pipfile b/Pipfile new file mode 100644 index 00000000000..0cd254c597f --- /dev/null +++ b/Pipfile @@ -0,0 +1,84 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +argcomplete = "==2.0.0" +attrs = "==22.1.0" +aws-xray-sdk = "==2.11.0" +bleach = "==5.0.1" +boto3 = "==1.26.30" +botocore = "==1.29.30" +certifi = "==2022.12.7" +cffi = "==1.15.1" +charset-normalizer = "==2.0.12" +click = "==8.1.3" +colorama = "==0.4.6" +coverage = {version = "==6.5.0", extras = ["toml"]} +cryptography = "==38.0.4" +docutils = "==0.17.1" +exceptiongroup = "==1.0.4" +execnet = "==1.9.0" +flake8 = "==3.9.2" +freezegun = "==1.2.2" +idna = "==3.4" +importlib-metadata = "==4.13.0" +importlib-resources = "==5.10.1" +iniconfig = "==1.1.1" +jaraco-classes = "==3.2.3" +jeepney = "==0.8.0" +jmespath = "==1.0.1" +jsonpatch = "==1.32" +jsonpointer = "==2.3" +jsonschema = "==4.17.3" +keyring = "==23.11.0" +mccabe = "==0.6.1" +mock = "==4.0.3" +more-itertools = "==9.0.0" +multidict = "==6.0.3" +pkginfo = "==1.9.2" +pkgutil-resolve-name = "==1.3.10" +placebo = "==0.9.0" +pluggy = "==1.0.0" +portalocker = "==2.6.0" +psutil = "==5.9.4" +pycodestyle = "==2.7.0" +pycparser = "==2.21" +pyflakes = "==2.3.1" +pygments = "==2.13.0" +pyrsistent = "==0.19.2" +pytest-cov = "==3.0.0" +pytest-recording = "==0.12.1" +pytest-sugar = "==0.9.6" +pytest-terraform = "==0.6.4" +pytest-xdist = "==3.1.0" +pytest = "==7.2.0" +python-dateutil = "==2.8.2" +pywin32-ctypes = "==0.2.0" +pywin32 = "==305" +pyyaml = "==6.0" +readme-renderer = "==37.3" +requests-toolbelt = "==0.10.1" +requests = "==2.27.1" +rfc3986 = "==2.0.0" +s3transfer = "==0.6.0" +secretstorage = "==3.3.3" +six = "==1.16.0" +tabulate = "==0.8.10" +termcolor = "==2.1.1" +tomli = "==2.0.1" +tqdm = "==4.64.1" +twine = "==3.8.0" +typing-extensions = "==4.4.0" +urllib3 = "==1.26.13" +vcrpy = "==4.2.1" +webencodings = "==0.5.1" +wrapt = "==1.14.1" +yarl = "==1.8.2" +zipp = "==3.11.0" + +[dev-packages] + +[requires] +python_version = "3.11" diff --git a/c7n/manager.py b/c7n/manager.py index d336a3027e5..2f292c2d0d3 100644 --- a/c7n/manager.py +++ b/c7n/manager.py @@ -16,6 +16,7 @@ resources = PluginRegistry('resources') from c7n.utils import dumps +from c7n.resource_metadata_update_with_email import call_api_and_update_resources def iter_filters(filters, block_end=False): @@ -119,7 +120,7 @@ def filter_resources(self, resources, event=None): # NOTE annotate resource ID property. moving this to query.py doesn't work. for r in resources: - if type(r) == dict and "c7n_resource_type_id" not in r: + if isinstance(r, dict) and "c7n_resource_type_id" not in r: try: r["c7n_resource_type_id"] = self.get_model().id except Exception as e: @@ -127,7 +128,18 @@ def filter_resources(self, resources, event=None): self.log.debug("Filtered from %d to %d %s" % ( original, len(resources), self.__class__.__name__.lower())) - return resources + + if not resources or len(resources) == 0: + # If resources is null or empty array, return resources as it is + return resources + else: + try: + updated_resources = call_api_and_update_resources(self, resources) + return updated_resources + except ValueError as error: + print(f"The resources will be returned without modifying the resource metadata for owner emails, as an error occurred: {error}") + # Return the original resources when an error occurs + return resources def get_model(self): """Returns the resource meta-model. diff --git a/c7n/resource_metadata_update_with_email.py b/c7n/resource_metadata_update_with_email.py new file mode 100644 index 00000000000..bb6fcf77a15 --- /dev/null +++ b/c7n/resource_metadata_update_with_email.py @@ -0,0 +1,102 @@ +# This module calls the API gateway, extracts email addresses based on resource tags, account, account's cost center information, and updates resource details with the relevant email data to enhance communication capabilities. + +import os +import json +import requests +import boto3 +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest + +PROVIDERS = { + "AWS": 0, + "Azure": 1, + "GCP": 2, +} + +def extract_appids(resource_list): + appids = {tag.get("Value") for resource in resource_list for tag in resource.get("Tags", []) if tag.get("Key") == "appid"} + return {"appid": list(appids)} + +def call_api_and_update_resources(self, resources, event=None): + try: + appids_data = extract_appids(resources) + # If the "appid" tag is missing , then skip the API call + # Used to avoid api calls for non-DJ accounts + if appids_data != []: + try: + # endpoint = os.environ.get('api_endpoint') + endpoint = 'https://ownerlookupapi.services.dowjones.io' + if not endpoint: + raise ValueError("API endpoint not defined in environment variables.") + + resource_path = '/service' + region = 'us-east-1' + service = 'execute-api' + + session = boto3.Session(region_name=region) + credentials = session.get_credentials() + + appids_data = extract_appids(resources) + if self.account_id: + appids_data["account"] = [self.account_id] + + request = AWSRequest(method='POST', url=endpoint + resource_path, headers={'Content-Type': 'application/json'}) + request.data = json.dumps(appids_data) + SigV4Auth(credentials, service, region).add_auth(request) + + response = requests.post( + request.url, + headers=request.headers, + data=request.data + ) + response.raise_for_status() + response_data = response.json() + + email_address = {} + def extract_from_dict(data_dict, parent_key=""): + nonlocal email_address + for key, value in data_dict.items(): + current_key = f"{parent_key}.{key}" if parent_key else key + if isinstance(value, dict): + extract_from_dict(value, parent_key=current_key) + elif isinstance(value, str) and "@" in value: + if parent_key not in email_address: + email_address[parent_key] = {} + email_address[parent_key][key] = value + + extract_from_dict(response_data) + + for resource in resources: + tags = resource.get("Tags", []) + app_id = next((tag.get("Value") for tag in tags if tag.get("Key") == "appid"), None) + owner_id = self.account_id + + if app_id: + app_email_data = email_address.get(f"appid.{app_id}", {}) + for key, value in app_email_data.items(): + if isinstance(value, str) and "@" in value: + resource[key] = value + + if owner_id: + email_data = email_address.get(f"account.{owner_id}", {}) + for key, value in email_data.items(): + if isinstance(value, str) and "@" in value: + resource[key] = value + + cost_cc_email_data = email_address.get(f"account.{owner_id}.cost_center_info", {}) + for key, value in cost_cc_email_data.items(): + if isinstance(value, str) and "@" in value: + resource[key] = value + + return resources + + except requests.exceptions.RequestException as req_error: + raise ValueError(f"Error making API request: {req_error}") + except (ValueError, json.JSONDecodeError) as json_error: + raise ValueError(f"Error processing API response: {json_error}") + except Exception as error: + raise ValueError(f"Unexpected error occurred: {error}") + except: + resources = resources + return resources + pass diff --git a/c7n/resources/asg.py b/c7n/resources/asg.py index 5cadcc4d6ba..019aca3f210 100644 --- a/c7n/resources/asg.py +++ b/c7n/resources/asg.py @@ -994,7 +994,8 @@ def process(self, asgs): # unless we were given a new value for min_size then # ensure it is at least as low as current_size update['MinSize'] = min(current_size, a['MinSize']) - elif type(self.data['desired-size']) == int: + elif isinstance(self.data['desired-size'], int): + update['DesiredCapacity'] = self.data['desired-size'] if update: diff --git a/c7n/resources/rds.py b/c7n/resources/rds.py index d6c4da49f34..dedc88ac5f2 100644 --- a/c7n/resources/rds.py +++ b/c7n/resources/rds.py @@ -1858,8 +1858,7 @@ def process(self, resources): if r.get( u['property'], jmespath.search( - self.conversion_map.get(u['property'], 'None'), r)) - != u['value']} + self.conversion_map.get(u['property'], 'None'), r)) != u['value']} if not param: continue param['ApplyImmediately'] = self.data.get('immediate', False) diff --git a/c7n/resources/vpc.py b/c7n/resources/vpc.py index aabb7d12319..39c550ac165 100644 --- a/c7n/resources/vpc.py +++ b/c7n/resources/vpc.py @@ -2885,63 +2885,3 @@ def process(self, resources, event=None): results.append(resource) return results - - -@Subnet.filter_registry.register('ip-allocation-threshold') -class SubnetIpAllocationFilter(Filter): - """Filters subnets based on ip allocation percentage - - :example: - - .. code-block:: yaml - - policies: - - name: subnet-ip-threshold-policy - resource: subnet - filters: - - type: ip-allocation-threshold - percentage: 80 - op: gte - """ - schema = type_schema( - 'ip-allocation-threshold', - percentage={'type': 'number'}, - op={'enum': ['eq', 'ne', 'lt', 'gt', 'lte', 'gte']} - ) - - permissions = ("ec2:DescribeSubnets",) - - def calculate_ip_allocation(self, subnet): - subnetMask = subnet.get('CidrBlock').split('/')[1] - hostBits = 32 - int(subnetMask) - totalHost = ((2 ** hostBits) - 2) - availableHost = subnet.get('AvailableIpAddressCount') - ipsUsed = totalHost - availableHost - percentageOfIpsUsed = (ipsUsed / totalHost) * 100 - return percentageOfIpsUsed - - def process(self, resources, event=None): - results = [] - threshold_percentage = self.data.get('percentage') - op = self.data.get('op') - for subnet in resources: - percentage_used = self.calculate_ip_allocation(subnet) - if op == 'eq': - if threshold_percentage == percentage_used: - results.append(subnet) - elif op == 'ne': - if threshold_percentage != percentage_used: - results.append(subnet) - elif op == 'lt': - if percentage_used < threshold_percentage: - results.append(subnet) - elif op == 'gt': - if percentage_used > threshold_percentage: - results.append(subnet) - elif op == 'lte': - if (percentage_used < threshold_percentage) or (percentage_used == threshold_percentage): - results.append(subnet) - elif op == 'gte': - if (percentage_used > threshold_percentage) or (percentage_used == threshold_percentage): - results.append(subnet) - return results diff --git a/tests/conftest.py b/tests/conftest.py index 3c14a12d58d..3d4dd7f8d59 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,7 +26,7 @@ class LazyPluginCacheDir: pass -pytest_plugins = ("pytest_recording",) +# pytest_plugins = ("pytest_recording",) # If we have C7N_FUNCTIONAL make sure Replay is False otherwise enable Replay LazyReplay.value = not strtobool(os.environ.get('C7N_FUNCTIONAL', 'no')) From 2341eb59b3a0a69e671b4603ff315cbe7316958c Mon Sep 17 00:00:00 2001 From: Gianncarlo Giannattasio Date: Tue, 12 Mar 2024 16:44:32 -0400 Subject: [PATCH 4/7] Added folder option to GCP --- tools/c7n_org/scripts/gcpprojects.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tools/c7n_org/scripts/gcpprojects.py b/tools/c7n_org/scripts/gcpprojects.py index 4a2463aeabb..0e27f6ec898 100644 --- a/tools/c7n_org/scripts/gcpprojects.py +++ b/tools/c7n_org/scripts/gcpprojects.py @@ -13,22 +13,38 @@ help="File to store the generated config (default stdout)") @click.option('-i', '--ignore', multiple=True, help="list of folders that won't be added to the config file") -def main(output, ignore): +@click.option('-fold', '--folders', required=False, multiple=True, + help="List folders the will be added to the config file") +@click.option('-ap','--appscript', default=False, is_flag=True, + help="list of app script projects to account files") +def main(output, ignore, appscript, folders): """ Generate a c7n-org gcp projects config file """ - client = Session().client('cloudresourcemanager', 'v1', 'projects') results = [] for page in client.execute_paged_query('list', {}): for project in page.get('projects', []): + # Exclude App Script GCP Projects + if appscript == False: + if 'sys-' in project['projectId']: + continue + if project['lifecycleState'] != 'ACTIVE': continue if project["parent"]["id"] in ignore: continue + + if folders != (): + if project["parent"]["type"] != "folder": + continue + for fold in folders: + if project["parent"]["id"] != fold: + continue + project_info = { 'project_id': project['projectId'], @@ -47,4 +63,4 @@ def main(output, ignore): if __name__ == '__main__': - main() + main() \ No newline at end of file From 4d627490cfcaa889866fb191a79db2e842d05d81 Mon Sep 17 00:00:00 2001 From: Gianncarlo Giannattasio Date: Tue, 12 Mar 2024 16:59:40 -0400 Subject: [PATCH 5/7] readd ip-allocation-threshold --- c7n/resources/vpc.py | 56 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/c7n/resources/vpc.py b/c7n/resources/vpc.py index 39c550ac165..d122f807ccc 100644 --- a/c7n/resources/vpc.py +++ b/c7n/resources/vpc.py @@ -2885,3 +2885,59 @@ def process(self, resources, event=None): results.append(resource) return results + +@Subnet.filter_registry.register('ip-allocation-threshold') +class SubnetIpAllocationFilter(Filter): + """Filters subnets based on ip allocation percentage + :example: + .. code-block:: yaml + policies: + - name: subnet-ip-threshold-policy + resource: subnet + filters: + - type: ip-allocation-threshold + percentage: 80 + op: gte + """ + schema = type_schema( + 'ip-allocation-threshold', + percentage={'type': 'number'}, + op={'enum': ['eq', 'ne', 'lt', 'gt', 'lte', 'gte']} + ) + + permissions = ("ec2:DescribeSubnets",) + + def calculate_ip_allocation(self, subnet): + subnetMask = subnet.get('CidrBlock').split('/')[1] + hostBits = 32 - int(subnetMask) + totalHost = ((2 ** hostBits) - 2) + availableHost = subnet.get('AvailableIpAddressCount') + ipsUsed = totalHost - availableHost + percentageOfIpsUsed = (ipsUsed / totalHost) * 100 + return percentageOfIpsUsed + + def process(self, resources, event=None): + results = [] + threshold_percentage = self.data.get('percentage') + op = self.data.get('op') + for subnet in resources: + percentage_used = self.calculate_ip_allocation(subnet) + if op == 'eq': + if threshold_percentage == percentage_used: + results.append(subnet) + elif op == 'ne': + if threshold_percentage != percentage_used: + results.append(subnet) + elif op == 'lt': + if percentage_used < threshold_percentage: + results.append(subnet) + elif op == 'gt': + if percentage_used > threshold_percentage: + results.append(subnet) + elif op == 'lte': + if (percentage_used < threshold_percentage) or (percentage_used == threshold_percentage): + results.append(subnet) + elif op == 'gte': + if (percentage_used > threshold_percentage) or (percentage_used == threshold_percentage): + results.append(subnet) + return results \ No newline at end of file From 8fb18592d32d236238966495b65cb378a24f4dff Mon Sep 17 00:00:00 2001 From: Wayne Cierkowski Date: Mon, 22 Apr 2024 12:37:40 -0400 Subject: [PATCH 6/7] IE-3315: Added filter to project query --- tools/c7n_org/scripts/gcpprojects.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/c7n_org/scripts/gcpprojects.py b/tools/c7n_org/scripts/gcpprojects.py index 4baefa453b1..ab50ee7bdbc 100644 --- a/tools/c7n_org/scripts/gcpprojects.py +++ b/tools/c7n_org/scripts/gcpprojects.py @@ -23,14 +23,12 @@ def main(output, exclude, buid, appscript): """ client = Session().client('cloudresourcemanager', 'v1', 'projects') + query_params = {'filter': f"parent.type:folder parent.id:{buid}"} if buid else {} results = [] - for page in client.execute_paged_query('list', {}): + for page in client.execute_paged_query('list', query_params): for project in page.get('projects', []): - if buid and project["parent"]["id"] != buid: - continue - # Exclude App Script GCP Projects if appscript == False: if 'sys-' in project['projectId']: From 5c06348ed14256e8bc8aebfa3110d58dd6f7c717 Mon Sep 17 00:00:00 2001 From: Gianncarlo Giannattasio Date: Tue, 4 Jun 2024 13:39:38 -0400 Subject: [PATCH 7/7] Ignore Closed Accounts OU --- tools/c7n_org/scripts/orgaccounts.py | 157 ++++++++++++++++----------- 1 file changed, 93 insertions(+), 64 deletions(-) diff --git a/tools/c7n_org/scripts/orgaccounts.py b/tools/c7n_org/scripts/orgaccounts.py index 5c567910689..799cd77d3d5 100644 --- a/tools/c7n_org/scripts/orgaccounts.py +++ b/tools/c7n_org/scripts/orgaccounts.py @@ -8,34 +8,54 @@ from c7n.resources.aws import ApiStats from c7n.credentials import assumed_session, SessionFactory from c7n.utils import yaml_dump +import re ROLE_TEMPLATE = "arn:aws:iam::{Id}:role/OrganizationAccountAccessRole" NAME_TEMPLATE = "{name}" -log = logging.getLogger('orgaccounts') +log = logging.getLogger("orgaccounts") @click.command() @click.option( - '--role', + "--role", default=ROLE_TEMPLATE, - help="Role template for accounts in the config, defaults to %s" % ROLE_TEMPLATE) + help="Role template for accounts in the config, defaults to %s" % ROLE_TEMPLATE, +) @click.option( - '--name', + "--name", default=NAME_TEMPLATE, - help="Name template for accounts in the config, defaults to %s" % NAME_TEMPLATE) -@click.option('--ou', multiple=True, default=["/"], - help="Only export the given subtrees of an organization") -@click.option('-r', '--regions', multiple=True, - help="If specified, set regions per account in config") -@click.option('--assume', help="Role to assume for Credentials") -@click.option('--profile', help="AWS CLI Profile to use for Credentials") + help="Name template for accounts in the config, defaults to %s" % NAME_TEMPLATE, +) @click.option( - '-f', '--output', type=click.File('w'), - help="File to store the generated config (default stdout)") -@click.option('-a', '--active', is_flag=True, default=False, help="Get only active accounts") -@click.option('-i', '--ignore', multiple=True, - help="list of accounts that won't be added to the config file") + "--ou", + multiple=True, + default=["/"], + help="Only export the given subtrees of an organization", +) +@click.option( + "-r", + "--regions", + multiple=True, + help="If specified, set regions per account in config", +) +@click.option("--assume", help="Role to assume for Credentials") +@click.option("--profile", help="AWS CLI Profile to use for Credentials") +@click.option( + "-f", + "--output", + type=click.File("w"), + help="File to store the generated config (default stdout)", +) +@click.option( + "-a", "--active", is_flag=True, default=False, help="Get only active accounts" +) +@click.option( + "-i", + "--ignore", + multiple=True, + help="list of accounts that won't be added to the config file", +) def main(role, name, ou, assume, profile, output, regions, active, ignore): """Generate a c7n-org accounts config file using AWS Organizations @@ -44,8 +64,8 @@ def main(role, name, ou, assume, profile, output, regions, active, ignore): """ logging.basicConfig(level=logging.INFO) - stats, session = get_session(assume, 'c7n-org', profile) - client = session.client('organizations') + stats, session = get_session(assume, "c7n-org", profile) + client = session.client("organizations") accounts = [] for path in ou: ou = get_ou_from_path(client, path) @@ -55,40 +75,41 @@ def main(role, name, ou, assume, profile, output, regions, active, ignore): for a in accounts: tags = [] - path_parts = a['Path'].strip('/').split('/') + path_parts = a["Path"].strip("/").split("/") for idx, _ in enumerate(path_parts): - tags.append("path:/%s" % "/".join(path_parts[:idx + 1])) + tags.append("path:/%s" % "/".join(path_parts[: idx + 1])) - for k, v in a.get('Tags', {}).items(): + for k, v in a.get("Tags", {}).items(): tags.append("{}:{}".format(k, v)) - a['OrgId'] = a['Arn'].split('/')[1] - if not role.startswith('arn'): - arn_role = "arn:aws:iam::{}:role/{}".format(a['Id'], role) + a["OrgId"] = a["Arn"].split("/")[1] + if not role.startswith("arn"): + arn_role = "arn:aws:iam::{}:role/{}".format(a["Id"], role) else: arn_role = role.format(**a) ainfo = { - 'account_id': a['Id'], - 'email': a['Email'], - 'display_name': a['Name'], - 'name': a['Name'], - 'org_id': a['OrgId'], - 'tags': tags, - 'role': arn_role} - ainfo['name'] = name.format(**ainfo) + "account_id": a["Id"], + "email": a["Email"], + "display_name": a["Name"], + "name": a["Name"], + "org_id": a["OrgId"], + "tags": tags, + "role": arn_role, + } + ainfo["name"] = name.format(**ainfo) if regions: - ainfo['regions'] = list(regions) - if 'Tags' in a and a['Tags']: - ainfo['vars'] = a['Tags'] + ainfo["regions"] = list(regions) + if "Tags" in a and a["Tags"]: + ainfo["vars"] = a["Tags"] results.append(ainfo) # log.info('api calls {}'.format(stats.get_metadata())) - print(yaml_dump({'accounts': results}), file=output) + print(yaml_dump({"accounts": results}), file=output) def get_session(role, session_name, profile): - region = os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1') + region = os.environ.get("AWS_DEFAULT_REGION", "eu-west-1") stats = ApiStats(Bag(), Config.empty()) if role: s = assumed_session(role, session_name, region=region) @@ -99,38 +120,41 @@ def get_session(role, session_name, profile): def get_ou_from_path(client, path): - ou = client.list_roots()['Roots'][0] + ou = client.list_roots()["Roots"][0] if path == "/": - ou['Path'] = path + ou["Path"] = path return ou - ou_pager = client.get_paginator('list_organizational_units_for_parent') - for part in path.strip('/').split('/'): + ou_pager = client.get_paginator("list_organizational_units_for_parent") + for part in path.strip("/").split("/"): found = False - for page in ou_pager.paginate(ParentId=ou['Id']): - for child in page.get('OrganizationalUnits'): - if child['Name'] == part: + for page in ou_pager.paginate(ParentId=ou["Id"]): + for child in page.get("OrganizationalUnits"): + if child["Name"] == part: found = True ou = child break if found: break if found is False: - raise ValueError( - "No OU named:%r found in path: %s" % ( - path, path)) - ou['Path'] = path + raise ValueError("No OU named:%r found in path: %s" % (path, path)) + ou["Path"] = path return ou def get_sub_ous(client, ou): results = [ou] - ou_pager = client.get_paginator('list_organizational_units_for_parent') - for sub_ou in ou_pager.paginate( - ParentId=ou['Id']).build_full_result().get( - 'OrganizationalUnits'): - sub_ou['Path'] = "/%s/%s" % (ou['Path'].strip('/'), sub_ou['Name']) + ignore_regex = r".*Closed Accounts.*" + ou_pager = client.get_paginator("list_organizational_units_for_parent") + for sub_ou in ( + ou_pager.paginate(ParentId=ou["Id"]) + .build_full_result() + .get("OrganizationalUnits") + ): + sub_ou["Path"] = "/%s/%s" % (ou["Path"].strip("/"), sub_ou["Name"]) + if re.search(ignore_regex, sub_ou["Path"]): + continue results.extend(get_sub_ous(client, sub_ou)) return results @@ -141,25 +165,30 @@ def get_accounts_for_ou(client, ou, active, recursive=True, ignoredAccounts=()): if recursive: ous = get_sub_ous(client, ou) - account_pager = client.get_paginator('list_accounts_for_parent') + account_pager = client.get_paginator("list_accounts_for_parent") for ou in ous: - for a in account_pager.paginate( - ParentId=ou['Id']).build_full_result().get( - 'Accounts', []): - a['Path'] = ou['Path'] - a['Tags'] = { - t['Key']: t['Value'] for t in - client.list_tags_for_resource(ResourceId=a['Id']).get('Tags', ())} - if a['Id'] in ignoredAccounts: + for a in ( + account_pager.paginate(ParentId=ou["Id"]) + .build_full_result() + .get("Accounts", []) + ): + a["Path"] = ou["Path"] + a["Tags"] = { + t["Key"]: t["Value"] + for t in client.list_tags_for_resource(ResourceId=a["Id"]).get( + "Tags", () + ) + } + if a["Id"] in ignoredAccounts: continue if active: - if a['Status'] == 'ACTIVE': + if a["Status"] == "ACTIVE": results.append(a) else: results.append(a) return results -if __name__ == '__main__': +if __name__ == "__main__": main()