From 8ddea34f68c383d6a3abeca94603fc076efc3ded Mon Sep 17 00:00:00 2001 From: Kevin Reynolds Date: Fri, 12 Apr 2024 09:58:42 -0400 Subject: [PATCH 1/2] ready to test --- README.md | 9 ++++--- UserTags.md | 53 ++++++++++++++++++++++++++++++++++++++ base/app/app.py | 67 +++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 112 insertions(+), 17 deletions(-) create mode 100644 UserTags.md diff --git a/README.md b/README.md index 2585648..89feac6 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,13 @@ Once you've inspected [the installer](./orijen-udf-install.sh) and the contents sudo curl -s https://raw.githubusercontent.com/f5devcentral/orijen-udf-service/main/orijen-udf-base-install.sh | bash ``` -## UDF Deployment Tags Needed +## UDF User Tags Needed -### Base +Please see [here](./UserTags.md) for formatting information. -- [X] LabID - Each XC lab has a unique GUID. This is passed into the deployment to determine what resources and permissions should be provisioned. -- [X] SQS - This is the SQS queue the Orijen Provisioning tool is watching. This value should be specified as a URL. +- [X] LabID - Each XC lab has a unique GUID. This is passed into the tool to determine what resources and permissions should be provisioned. +- [X] SQS_q - This is the SQS queue the Orijen provisioning tool is watching. +- [X] SQS_r - This is the SQS region. ## Project Orijen diff --git a/UserTags.md b/UserTags.md new file mode 100644 index 0000000..0976dda --- /dev/null +++ b/UserTags.md @@ -0,0 +1,53 @@ +# UDF UserTag Values + +This tool uses UDF User tags to pass information on which lab is being launched and which SQS queue should be used to kick off the lab automation. + +UDF User Tags, applied to instances, must be < 64 **alphanumeric** characters. +Because this tool needs to pass information such as URLs to the metadata service, these strings will be base64 encoded with the padding removed. + +## Padding + +Here's an example function to generate tag values: +```python +import base64 + +def tag_value(s: str) -> str: + i_bytes = s.encode('utf-8') + e_bytes = base64.b64encode(i_bytes) + e_string = e_bytes.decode('utf-8') + tag_string = e_string.rstrip('=') + if len(tag_string) > 64: + raise ValueError(f"String len too long: {len(tag_string)}.") + return tag_string +``` + +The padding is added back to these strings before decoding in [``b64_lazy_decode``](./base/app/app.py). + +## SQS Values + +Standard ARN and URL formatting for SQS queues, once encoded to our tagging format, will bump up against the tagging character limit. +The tool expects 2 tags to be passed into the instance to account for this. + +```python + +def arn_tag_splitter(arn: str) -> str: + parts = arn.split(':') + if ( + len(parts) != 6 + or parts[0] != 'arn' + or parts[1] != 'aws' + or parts[2] != 'sqs' + ): + raise ValueError("Invalid ARN format.") + region = parts[3] + account = parts[4] + queue = parts[5] + return { + "SQS_r": tag_value(region), + "SQS_q": tag_value(f"{account}/{queue}") + } +``` + +## New Labs + +Please reach out to the project owners if you need a new lab created or have questions. \ No newline at end of file diff --git a/base/app/app.py b/base/app/app.py index 9c94994..76eb556 100644 --- a/base/app/app.py +++ b/base/app/app.py @@ -4,9 +4,21 @@ import json import re import atexit +import base64 import requests import boto3 +def b64_lazy_decode(s: str) -> str|None: + """ + Add padding (=) back and decode. + Necessary as UDF user tags only support alphanumeric characters + """ + try: + this = base64.b64decode(s + "=" * ((4 - len(s)) % 4)) + return this.decode('utf-8').rstrip('\n') + except Exception as e: + return None + def fetch_metadata(url: str, max_retries=5) -> dict|None: """ Fetch metadata. @@ -39,8 +51,41 @@ def find_aws_cred(cloud_accounts: dict) -> dict|None: return credential except Exception as e: return None + +def find_user_tags(meta_tags: list) -> dict|None: + """ + Find user_tags from instance metadata. + Return a dict with all b64 decoded tags. + """ + try: + all_tags = meta_tags[0].get("userTags", []) + tags = ["LabID", "SQS_r", "SQS_q"] + user_tags = {} + tag_list = [t for t in all_tags if t.get("name") in tags] + for tag in tag_list: + user_tags[tag["name"]] = b64_lazy_decode(tag["value"]) + except Exception as e: + return None + if len(user_tags) == 3: + return user_tags + else: + return None + +def build_sqs_url(region: str, q: str) -> str|None: + """ + Build a complete SQS queue URL from the pieces in user_tags + """ + try: + url = f"https://sqs/{region}.amazonaws.com/{q}" + return url + except Exception as e: + return None def find_sqs_region(url: str) -> str|None: + """ + Determine SQS region from URL. + Boto3 needs this regardless of region in URL. + """ try: region = re.search(r'sqs\.([\w-]+)\.amazonaws\.com', url).group(1) return region @@ -53,7 +98,7 @@ def query_metadata(metadata_base_url: str) -> dict|None: Retrieve AWS secret, AWS key, SQS URL, Lab GUID, deployer, deploy ID, and region. """ deployment_url = f"{metadata_base_url}/deployment" - deployment_tags_url = f"{metadata_base_url}/deploymentTags" + user_tags_url = f"{metadata_base_url}/userTags/name/XC/value/true" cloud_accounts_url = f"{metadata_base_url}/cloudAccounts" deployment = fetch_metadata(deployment_url) @@ -61,27 +106,23 @@ def query_metadata(metadata_base_url: str) -> dict|None: print("Unable to find deployment data.") return None - deployment_tags = fetch_metadata(deployment_tags_url) - if deployment_tags is None: - print("Unable to find deployment tags.") + user_tags = find_user_tags(fetch_metadata(user_tags_url)) + if user_tags is None: + print("Unable to find user tags.") return None aws_credential = find_aws_cred(fetch_metadata(cloud_accounts_url)) if aws_credential is None: print("Unable to find AWS metadata.") return None - - region = find_sqs_region(deployment_tags.get("SQS")) - if region is None: - print("Unable to find SQS region.") - return None - + try: dep_id = deployment.get("deployment")["id"] deployer = deployment.get("deployment")["deployer"] - lab_id = deployment_tags.get("LabID") - sqs_url = deployment_tags.get("SQS") - + lab_id = user_tags.get("LabID") + sqs_url = build_sqs_url(user_tags.get("SQS_r"), user_tags.get("SQS_q")) + region = find_sqs_region(sqs_url) + return { "depID": dep_id, "deployer": deployer, From 1af445d9c4b6e3aefc9613226adf52f4e084ed27 Mon Sep 17 00:00:00 2001 From: Kevin Reynolds Date: Fri, 12 Apr 2024 10:10:56 -0400 Subject: [PATCH 2/2] ready to merge --- README.md | 1 + base/app/app.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 89feac6..1e55ce6 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ Please see [here](./UserTags.md) for formatting information. - [X] LabID - Each XC lab has a unique GUID. This is passed into the tool to determine what resources and permissions should be provisioned. - [X] SQS_q - This is the SQS queue the Orijen provisioning tool is watching. - [X] SQS_r - This is the SQS region. +- [X] XC - This is used to identify the instance running the tool. It's value should be "true". ## Project Orijen diff --git a/base/app/app.py b/base/app/app.py index 76eb556..0e989f6 100644 --- a/base/app/app.py +++ b/base/app/app.py @@ -89,7 +89,7 @@ def find_sqs_region(url: str) -> str|None: try: region = re.search(r'sqs\.([\w-]+)\.amazonaws\.com', url).group(1) return region - except (AttributeError) as e: + except AttributeError as e: return None def query_metadata(metadata_base_url: str) -> dict|None: