From 5189cba66f8a8107ac932ffbd8962afee0dfa514 Mon Sep 17 00:00:00 2001 From: Wayne Cierkowski Date: Mon, 28 Oct 2024 13:34:11 -0400 Subject: [PATCH 1/5] New gcpprojects.py --- tools/c7n_org/scripts/gcpprojects.py | 92 ++++++++++++++++------------ 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/tools/c7n_org/scripts/gcpprojects.py b/tools/c7n_org/scripts/gcpprojects.py index fbd106955dd..f832ceff226 100644 --- a/tools/c7n_org/scripts/gcpprojects.py +++ b/tools/c7n_org/scripts/gcpprojects.py @@ -1,12 +1,10 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 +from c7n_gcp.client import Session import click import yaml -from c7n_gcp.client import Session - - @click.command() @click.option( "-f", @@ -19,66 +17,84 @@ "-e", "--exclude", multiple=True, - help="list of folders that won't be added to the config file", + help="List of folders that won't be added to the config file", ) @click.option( "-b", "--buid", required=False, multiple=True, - help="List folders the will be added to the config file", + help="List of folder IDs that will be added to the config file", ) @click.option( "-ap", "--appscript", default=False, is_flag=True, - help="list of app script projects to account files", + help="Exclude App Script projects from the account files", ) def main(output, exclude, appscript, buid): """ - Generate a c7n-org gcp projects config file + Generate a c7n-org GCP projects config file. """ - client = Session().client("cloudresourcemanager", "v1", "projects") + session = Session() + client = session.client("cloudresourcemanager", "v1", "projects") + folder_client = session.client("cloudresourcemanager", "v2", "folders") + + # Helper function to retrieve all subfolders for each buid + def get_all_subfolders(folder_id): + subfolders = set() + request = folder_client.execute_query("list", {"parent": f"folders/{folder_id}"}) + for folder in request.get("folders", []): + subfolders.add(folder["name"].split("/")[-1]) + subfolders.update(get_all_subfolders(folder["name"].split("/")[-1))) # Recursive call + return subfolders + + # Check if buid is empty; if so, assume flat structure and set organization ID + if not buid: + print("No BUID specified; assuming flat organization. Listing all projects under organization.") + organization_id = "organizations/161588151302" # Sandbox organization ID + all_folders = set() + else: + # Gather all folders and subfolders for hierarchical structure + all_folders = set(buid) + for folder_id in buid: + all_folders.update(get_all_subfolders(folder_id)) results = [] for page in client.execute_paged_query("list", {}): for project in page.get("projects", []): - # Exclude App Script GCP Projects - if appscript == False: - if "sys-" in project["projectId"]: - continue - - if ( - project["lifecycleState"] != "ACTIVE" - or project["projectNumber"] in exclude - ): + # Exclude App Script projects if the flag is set + if not appscript and "sys-" in project["projectId"]: continue - if buid != (): - if project["parent"]["type"] != "folder": - continue - for fold in buid: - if project["parent"]["id"] != fold: - continue - - project_info = { - "project_id": project["projectId"], - "project_number": project["projectNumber"], - "name": project["name"], - } + # Exclude projects in inactive states or those in excluded folders + if project["lifecycleState"] != "ACTIVE" or project["projectNumber"] in exclude: + continue - if "labels" in project: - project_info["tags"] = [ - "%s:%s" % (k, v) for k, v in project.get("labels", {}).items() - ] - project_info["vars"] = { - k: v for k, v in project.get("labels", {}).items() + # Determine if project is under specified folders or directly under organization + if (not buid and project["parent"].get("type") == "organization" and project["parent"].get("id") == organization_id) or \ + (buid and project["parent"].get("type") == "folder" and project["parent"].get("id") in all_folders): + # Collect project details + project_info = { + "project_id": project["projectId"], + "project_number": project["projectNumber"], + "name": project["name"], } - results.append(project_info) - output.write(yaml.safe_dump({"projects": results}, default_flow_style=False)) + # Include labels if they exist + if "labels" in project: + project_info["tags"] = [ + f"{k}:{v}" for k, v in project.get("labels", {}).items() + ] + project_info["vars"] = { + k: v for k, v in project.get("labels", {}).items() + } + + results.append(project_info) + # Output project information to YAML + output.write(yaml.safe_dump({"projects": results}, default_flow_style=False)) if __name__ == "__main__": - main() \ No newline at end of file + main() From 11e52768829d0aaf34ef00e6b588fed707f864f3 Mon Sep 17 00:00:00 2001 From: Wayne Cierkowski Date: Mon, 28 Oct 2024 13:46:32 -0400 Subject: [PATCH 2/5] test --- tools/c7n_org/scripts/gcpprojects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/c7n_org/scripts/gcpprojects.py b/tools/c7n_org/scripts/gcpprojects.py index f832ceff226..e780a7f0638 100644 --- a/tools/c7n_org/scripts/gcpprojects.py +++ b/tools/c7n_org/scripts/gcpprojects.py @@ -47,7 +47,7 @@ def get_all_subfolders(folder_id): request = folder_client.execute_query("list", {"parent": f"folders/{folder_id}"}) for folder in request.get("folders", []): subfolders.add(folder["name"].split("/")[-1]) - subfolders.update(get_all_subfolders(folder["name"].split("/")[-1))) # Recursive call + subfolders.update(get_all_subfolders(folder["name"].split("/")[-1])) # Recursive call return subfolders # Check if buid is empty; if so, assume flat structure and set organization ID From 32b3ee0017ebebb4b857c0e2a55c876a1ba09712 Mon Sep 17 00:00:00 2001 From: Wayne Cierkowski Date: Tue, 29 Oct 2024 06:33:34 -0400 Subject: [PATCH 3/5] Revised on flat organization --- tools/c7n_org/scripts/gcpprojects.py | 67 ++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/tools/c7n_org/scripts/gcpprojects.py b/tools/c7n_org/scripts/gcpprojects.py index e780a7f0638..1f0651b190a 100644 --- a/tools/c7n_org/scripts/gcpprojects.py +++ b/tools/c7n_org/scripts/gcpprojects.py @@ -51,30 +51,23 @@ def get_all_subfolders(folder_id): return subfolders # Check if buid is empty; if so, assume flat structure and set organization ID + all_folders = set() if not buid: print("No BUID specified; assuming flat organization. Listing all projects under organization.") - organization_id = "organizations/161588151302" # Sandbox organization ID - all_folders = set() - else: - # Gather all folders and subfolders for hierarchical structure - all_folders = set(buid) - for folder_id in buid: - all_folders.update(get_all_subfolders(folder_id)) + organization_id = "organizations/161588151302" # Replace with your actual organization ID - results = [] - for page in client.execute_paged_query("list", {}): - for project in page.get("projects", []): - # Exclude App Script projects if the flag is set - if not appscript and "sys-" in project["projectId"]: - continue + # Fetch projects directly under the organization + results = [] + for page in client.execute_paged_query("list", {"parent": organization_id}): + for project in page.get("projects", []): + # Exclude App Script projects if the flag is set + if not appscript and "sys-" in project["projectId"]: + continue - # Exclude projects in inactive states or those in excluded folders - if project["lifecycleState"] != "ACTIVE" or project["projectNumber"] in exclude: - continue + # Exclude projects in inactive states or those in excluded folders + if project["lifecycleState"] != "ACTIVE" or project["projectNumber"] in exclude: + continue - # Determine if project is under specified folders or directly under organization - if (not buid and project["parent"].get("type") == "organization" and project["parent"].get("id") == organization_id) or \ - (buid and project["parent"].get("type") == "folder" and project["parent"].get("id") in all_folders): # Collect project details project_info = { "project_id": project["projectId"], @@ -93,6 +86,42 @@ def get_all_subfolders(folder_id): results.append(project_info) + else: + # Hierarchical structure: gather all folders and subfolders for each buid + for folder_id in buid: + all_folders.update(get_all_subfolders(folder_id)) + + results = [] + for page in client.execute_paged_query("list", {}): + for project in page.get("projects", []): + # Exclude App Script projects if the flag is set + if not appscript and "sys-" in project["projectId"]: + continue + + # Exclude projects in inactive states or those in excluded folders + if project["lifecycleState"] != "ACTIVE" or project["projectNumber"] in exclude: + continue + + # Only include projects within specified folders or subfolders + if project["parent"].get("type") == "folder" and project["parent"].get("id") in all_folders: + # Collect project details + project_info = { + "project_id": project["projectId"], + "project_number": project["projectNumber"], + "name": project["name"], + } + + # Include labels if they exist + if "labels" in project: + project_info["tags"] = [ + f"{k}:{v}" for k, v in project.get("labels", {}).items() + ] + project_info["vars"] = { + k: v for k, v in project.get("labels", {}).items() + } + + results.append(project_info) + # Output project information to YAML output.write(yaml.safe_dump({"projects": results}, default_flow_style=False)) From 8f7237482925e108a9da9d56817a82f963a550bb Mon Sep 17 00:00:00 2001 From: Wayne Cierkowski Date: Tue, 29 Oct 2024 06:44:03 -0400 Subject: [PATCH 4/5] Test --- tools/c7n_org/scripts/gcpprojects.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/c7n_org/scripts/gcpprojects.py b/tools/c7n_org/scripts/gcpprojects.py index 1f0651b190a..22f77f5a017 100644 --- a/tools/c7n_org/scripts/gcpprojects.py +++ b/tools/c7n_org/scripts/gcpprojects.py @@ -50,14 +50,13 @@ def get_all_subfolders(folder_id): subfolders.update(get_all_subfolders(folder["name"].split("/")[-1])) # Recursive call return subfolders - # Check if buid is empty; if so, assume flat structure and set organization ID - all_folders = set() + results = [] if not buid: + # No buid provided; assuming a flat organization structure print("No BUID specified; assuming flat organization. Listing all projects under organization.") organization_id = "organizations/161588151302" # Replace with your actual organization ID # Fetch projects directly under the organization - results = [] for page in client.execute_paged_query("list", {"parent": organization_id}): for project in page.get("projects", []): # Exclude App Script projects if the flag is set @@ -88,10 +87,10 @@ def get_all_subfolders(folder_id): else: # Hierarchical structure: gather all folders and subfolders for each buid + all_folders = set(buid) for folder_id in buid: all_folders.update(get_all_subfolders(folder_id)) - results = [] for page in client.execute_paged_query("list", {}): for project in page.get("projects", []): # Exclude App Script projects if the flag is set From c166787b28eddbc60ba83aa473b7f39841e4d6d9 Mon Sep 17 00:00:00 2001 From: Wayne Cierkowski Date: Tue, 29 Oct 2024 06:50:21 -0400 Subject: [PATCH 5/5] test --- tools/c7n_org/scripts/gcpprojects.py | 69 +++++++++------------------- 1 file changed, 22 insertions(+), 47 deletions(-) diff --git a/tools/c7n_org/scripts/gcpprojects.py b/tools/c7n_org/scripts/gcpprojects.py index 22f77f5a017..b9eea3fa0d3 100644 --- a/tools/c7n_org/scripts/gcpprojects.py +++ b/tools/c7n_org/scripts/gcpprojects.py @@ -50,23 +50,34 @@ def get_all_subfolders(folder_id): subfolders.update(get_all_subfolders(folder["name"].split("/")[-1])) # Recursive call return subfolders + # Check if buid is empty; if so, assume flat structure and set organization ID results = [] + organization_id = "161588151302" # Replace with your actual organization ID + all_folders = set(buid) if buid else set() + if not buid: - # No buid provided; assuming a flat organization structure print("No BUID specified; assuming flat organization. Listing all projects under organization.") - organization_id = "organizations/161588151302" # Replace with your actual organization ID - # Fetch projects directly under the organization - for page in client.execute_paged_query("list", {"parent": organization_id}): - for project in page.get("projects", []): - # Exclude App Script projects if the flag is set - if not appscript and "sys-" in project["projectId"]: - continue + else: + # Hierarchical structure: gather all folders and subfolders for each buid + for folder_id in buid: + all_folders.update(get_all_subfolders(folder_id)) - # Exclude projects in inactive states or those in excluded folders - if project["lifecycleState"] != "ACTIVE" or project["projectNumber"] in exclude: - continue + # Retrieve all projects and apply filtering in Python + for page in client.execute_paged_query("list", {}): + for project in page.get("projects", []): + # Exclude App Script projects if the flag is set + if not appscript and "sys-" in project["projectId"]: + continue + # Exclude projects in inactive states or those in excluded folders + if project["lifecycleState"] != "ACTIVE" or project["projectNumber"] in exclude: + continue + + # Filtering logic for flat and hierarchical organization structures + if (not buid and project["parent"].get("type") == "organization" and project["parent"].get("id") == organization_id) or \ + (buid and project["parent"].get("type") == "folder" and project["parent"].get("id") in all_folders): + # Collect project details project_info = { "project_id": project["projectId"], @@ -85,42 +96,6 @@ def get_all_subfolders(folder_id): results.append(project_info) - else: - # Hierarchical structure: gather all folders and subfolders for each buid - all_folders = set(buid) - for folder_id in buid: - all_folders.update(get_all_subfolders(folder_id)) - - for page in client.execute_paged_query("list", {}): - for project in page.get("projects", []): - # Exclude App Script projects if the flag is set - if not appscript and "sys-" in project["projectId"]: - continue - - # Exclude projects in inactive states or those in excluded folders - if project["lifecycleState"] != "ACTIVE" or project["projectNumber"] in exclude: - continue - - # Only include projects within specified folders or subfolders - if project["parent"].get("type") == "folder" and project["parent"].get("id") in all_folders: - # Collect project details - project_info = { - "project_id": project["projectId"], - "project_number": project["projectNumber"], - "name": project["name"], - } - - # Include labels if they exist - if "labels" in project: - project_info["tags"] = [ - f"{k}:{v}" for k, v in project.get("labels", {}).items() - ] - project_info["vars"] = { - k: v for k, v in project.get("labels", {}).items() - } - - results.append(project_info) - # Output project information to YAML output.write(yaml.safe_dump({"projects": results}, default_flow_style=False))