Feature: AwsServicePrincipalOsduClient (#36)

* Added: AwsServicePrincipalOsduClient * Added: _service_prinicpal_util.py from AWS with revisions documented in module comments. * Added: unit test for AwsServicePrincipalOsduClient * Updated client/__init__.py to import all of the concrete client classes into the top-level package namespace for cleaner importing. * Renamed: client/base.py --> _base.py as it should not be imported from outside of package. * README updates * Added: integration tests * Updated version
pariveda · Feb 25, 2022 · 9dfddd2 · 9dfddd2
1 parent cfda3ce
commit 9dfddd2
Show file tree

Hide file tree

Showing 10 changed files with 297 additions and 59 deletions.
diff --git a/README.md b/README.md
@@ -6,12 +6,14 @@ A simple python client for the [OSDU](https://community.opengroup.org/osdu) data
 
 - [Clients](#clients)
   - [SimpleOsduClient](#simpleosduclient)
+  - [AwsServicePrincipalOsduClient](#awsserviceprincipalosduclient)
   - [AwsOsduClient](#awsosduclient)
 - [Currently supported methods](#currently-supported-methods)
 - [Installation](#installation)
 - [Tests](#tests)
 - [Usage](#usage)
   - [Instantiating the SimpleOsduClient](#instantiating-the-simpleosduclient)
+  - [Instantiating the AwsServicePrincipalOsduClient](#instantiating-the-awsosduclient)
   - [Instantiating the AwsOsduClient](#instantiating-the-awsosduclient)
   - [Using the client](#using-the-client)
     - [Search for records by query](#search-for-records-by-query)
@@ -37,19 +39,30 @@ login form or otheer mechanism. With this SimpleOsduClient, you simply provide t
 With this simplicity, you are also then respnsible for reefreeshing the token as needed and
 re-instantiating the client with the new token.
 
+### AwsOsduServicePrincipalClient
+
+**Requires**: `boto3==1.15.*`
+
+Good for batch tasks that don't have an interactive front-end. Token management is handled
+with the boto3 library directly through the Cognito service. You have to supply additional arguments for this.
+
+For OSDU on AWS, this client is usually simpler than the AwsOsduClient as long as you have IAM credentials to access the necessary resources. You only need to provide the OSDU resource_prefix, region, and profile.
+
 ### AwsOsduClient
 
 **Requires**: `boto3==1.15.*`
 
 Good for batch tasks that don't have an interactive front-end. Token management is handled
 with the boto3 library directly through the Cognito service. You have to supply additional arguments for this.
 
+For OSDU on AWS, this client is useful in the case where you may want to perform actions as a specific OSDU user rather than as the ServicePrinicpal.
+
 ## Currently supported methods
 
-- [search](osdu/search.py)
+- [search](osdu/services/search.py)
   - query
   - query_with_paging
-- [storage](osdu/storage.py)
+- [storage](osdu/services/storage.py)
   - query_all_kinds
   - get_record
   - get_records
@@ -58,13 +71,13 @@ with the boto3 library directly through the Cognito service. You have to supply
   - store_records
   - delete_record
   - purge_record
-- [dataset](osdu/dataset.py)
+- [dataset](osdu/services/dataset.py)
   - get_dataset_registry
   - get_dataset_registries
   - get_storage_instructions
   - register_dataset
   - get_retrieval_instructions
-- [entitlement](osdu/entitlement.py)
+- [entitlements](osdu/services/entitlements.py)
   - get_groups
   - get_group_members
   - add_group_member
@@ -98,18 +111,34 @@ python -m unittest -v tests.integration
 If environment variable `OSDU_API_URL` is set, then it does not need to be passed as an argument. Otherwise it must be passed as keyword argument.
 
 ```python
-from osdu.client.simple import SimpleOsduClient
+from osdu.client import SimpleOsduClient
 
-data_partition = 'opendes'
+data_partition = 'osdu'
 token = 'token-received-from-front-end-app'
 
 # With env var `OSDU_API_URL` set in current environment.
-osdu = SimpleOsduClient(data_partition, token)
+osdu_client = SimpleOsduClient(data_partition, token)
 
 # Without env var set.
 api_url = 'https://your.api.base_url.com'
-osdu = SimpleOsduClient(data_partition, token, api_url=api_url)
+osdu_client = SimpleOsduClient(data_partition, token, api_url=api_url)
+
+```
+
+### Instantiating the AwsServicePrincipalOsduClient
 
+```python
+from osdu.client import AwsOsduClient
+
+data_partition = 'osdu'
+resource_prefix = 'osdur3mX'
+
+osdu_client = AwsServicePrincipalOsduClient(
+    data_partition,
+    resource_prefix,
+    profile=os.environ['AWS_PROFILE'],
+    region=os.environ['AWS_DEFAULT_REGION']
+)
 ```
 
 ### Instantiating the AwsOsduClient
@@ -125,18 +154,18 @@ Environment variables:
 1. `AWS_SECRETHASH`
 
 ```python
-from osdu.client.aws import AwsOsduClient
+from osdu.client import AwsOsduClient
 
 data_partition = 'osdu'
 
-osdu = AwsOsduClient(data_partition)
+osdu_client = AwsOsduClient(data_partition)
 ```
 
 If you have not set the above environment variales—or you have only set some—then you will need to pass any undefined as args when instantiating the client.
 
 ```python
 from getpass import getpass
-from osdu.client.aws import AwsOsduClient
+from osdu.client import AwsOsduClient
 
 api_url = 'https://your.api.url.com'  # Must be base URL only
 client_id = 'YOURCLIENTID'
@@ -152,7 +181,7 @@ secretHash = base64.b64encode(dig).decode()
 
 
 
-osdu = AwsOsduClient(data_partition,
+osdu_client = AwsOsduClient(data_partition,
     api_url=api_url,
     client_id=client_id,
     user=user,
@@ -169,9 +198,9 @@ Below are just a few usage examples. See [integration tests](https://github.com/
 
 ```python
 query = {
-    "kind": f"opendes:osdu:*:*"
+    "kind": f"osdu:wks:*:*"
 }
-result = osdu.search.query(query)
+result = osdu_client.search.query(query)
 # { results: [ {...}, .... ], totalCount: ##### }
 ```
 
@@ -182,10 +211,10 @@ For result sets larger than 1,000 records, use the query with paging method.
 ```python
 page_size = 100 # Number of records per page (1-1000)
 query = {
-    "kind": f"opendes:osdu:*:*",
+    "kind": f"osdu:wks:*:*",
     "limit": page_size
 }
-result = osdu.search.query_with_paging(query)
+result = osdu_client.search.query_with_paging(query)
 
 # Iterate over the pages to do something with the results.
 for page, total_count in result:
@@ -197,7 +226,7 @@ for page, total_count in result:
 
 ```python
 record_id = 'opendes:doc:123456789'
-result = osdu.storage.get_record(record_id)
+result = osdu_client.storage.get_record(record_id)
 # { 'id': 'opendes:doc:123456789', 'kind': ..., 'data': {...}, 'acl': {...}, .... }
 ```
 
@@ -208,14 +237,14 @@ new_or_updated_record = './record-123.json'
 with open(new_or_updated_record, 'r') as _file:
     record = json.load(_file)
 
-result = osdu.storage.store_records([record])
+result = osdu_client.storage.store_records([record])
 
 ```
 
 #### List groupmembership for the current user
 
 ```python
-result = osduClient.entitlements.get_groups()
+result = osdu_client.entitlements.get_groups()
 # {
 #  "desId": "[email protected]",
 #  "groups": [
@@ -243,7 +272,7 @@ result = osduClient.entitlements.get_groups()
 ### List membership of a particular group
 
 ```python
-result = osduClient.entitlements.get_group_members('[email protected]')
+result = osdu_client.entitlements.get_group_members('[email protected]')
 #{
 #  "members": [
 #    {
@@ -272,13 +301,13 @@ query =  {
      #OWNER or MEMBER
      "role": "MEMBER",
  }
-result = osduClient.entitlements.add_group_member('[email protected]',query)
+result = osdu_client.entitlements.add_group_member('[email protected]',query)
 query =  {
      "email": "[email protected]",
      #OWNER or MEMBER
      "role": "OWNER",
  }
-result = osduClient.entitlements.add_group_member('[email protected]',query)
+result = osdu_client.entitlements.add_group_member('[email protected]',query)
 ```
 
 ### Delete user from a particular group
@@ -291,11 +320,11 @@ query =  {
      #OWNER or MEMBER
      "role": "MEMBER",
  }
-result = osduClient.entitlements.delete_group_member('[email protected]',query)
+result = osdu_client.entitlements.delete_group_member('[email protected]',query)
 query =  {
      "email": "[email protected]",
      #OWNER or MEMBER
      "role": "OWNER",
  }
-result = osduClient.entitlements.delete_group_member('[email protected]',query)
+result = osdu_client.entitlements.delete_group_member('[email protected]',query)
 ```
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.3.0
+0.4.0
diff --git a/osdu/client/__init__.py b/osdu/client/__init__.py
@@ -1 +1,3 @@
-__all__ = ['aws', 'simple']
+from osdu.client.aws import AwsOsduClient
+from osdu.client.simple import SimpleOsduClient
+from osdu.client.aws_service_principal import AwsServicePrincipalOsduClient
diff --git a/osdu/client/base.py → osdu/client/_base.py b/osdu/client/base.py → osdu/client/_base.py
@@ -54,7 +54,10 @@ def __init__(self, data_partition_id, api_url: str = None):
         """
         self._data_partition_id = data_partition_id
         # TODO: Validate api_url against URL regex pattern.
-        self._api_url = (api_url or os.environ.get('OSDU_API_URL')).rstrip('/')
+        api_url = api_url or os.environ.get('OSDU_API_URL')
+        if not api_url:
+            raise Exception('No API URL found.')
+        self._api_url = api_url.rstrip('/')
 
         # Instantiate services.
         self._search = SearchService(self)
@@ -64,7 +67,3 @@ def __init__(self, data_partition_id, api_url: str = None):
         # TODO: Implement these services.
         # self.__legal = LegaService(self)
 
-    # Abstract Method
-    def get_tokens(self, password):
-        raise NotImplementedError(
-            'This method must be implemented by a subclass')
diff --git a/osdu/client/_service_principal_util.py b/osdu/client/_service_principal_util.py
@@ -0,0 +1,119 @@
+# Copyright © 2020 Amazon Web Services
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# ===================================
+# REVISIONS
+# ===================================
+#
+# Date          Author & Description
+# ---------     ---------------------
+# 2022-02-23    [email protected]
+#               - Updated constructor to optionally accept AWS profile and region instead of
+#                 a boto3 session.
+#               - Constructor to require resource_prefix and make this an instance variable.
+#               - Refactored _get_secret method to fix UnboundLocalError for local variable 'secret'.
+#               - Refactored _get_secret method to simplify try/except flow and to print secret_name on exception.
+#               - Updated formatting to be PEP8-compliant.
+#
+import base64
+import boto3
+import requests
+import json
+import botocore.exceptions
+
+
+class ServicePrincipalUtil:
+
+    @property
+    def api_url(self):
+        return self._api_url
+
+    def __init__(
+            self,
+            resource_prefix: str,
+            aws_session: boto3.Session = None,
+            region: str = None,
+            profile: str = None
+    ):
+        """If a session is not provided, then region and profile must be provided. 
+        If none of these are provided, then boto3.Session will check for env vars: AWS_PROFILE and AWS_DEFAULT_REGION. 
+        If not found there, then instantiation will fail.
+
+        :param resource_prefix: Resource prefix from OSDU deployment. e.g. 'osdur3mX'
+        :param aws_session: boto3 sesssion to use for retrieving paramaeters an secrets for the OSDU instance.
+        :param region:  AWS Region where OSDU instance is deployed. e.g. 'us-east-1'
+        :param profile: AWS credentials (CLI) profile name.
+        """
+        # If a boto session is provided, then use it. Otherwise, instantiate a new one with provided
+        # region and profile.
+        if aws_session:
+            self._session = aws_session
+        else:
+            self._session = boto3.Session(
+                region_name=region, profile_name=profile)
+        self._api_url = self._get_ssm_parameter(
+            f'/osdu/{resource_prefix}/api/url')
+
+    def _get_ssm_parameter(self, ssm_path):
+        ssm_client = self._session.client('ssm')
+        ssm_response = ssm_client.get_parameter(Name=ssm_path)
+        return ssm_response['Parameter']['Value']
+
+    def _get_secret(self, secret_name, secret_dict_key):
+        client = self._session.client(service_name='secretsmanager')
+        # In this sample we only handle the specific exceptions for the 'GetSecretValue' API.
+        # See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html
+        try:
+            secret_response = client.get_secret_value(SecretId=secret_name)
+            secret_val = None
+            if 'SecretString' in secret_response:
+                secret_val = secret_response['SecretString']
+            elif 'SecretBinary' in secret_response:
+                secret_val = base64.b64decode(secret_response['SecretBinary'])
+            secret_json = json.loads(secret_val)[secret_dict_key]
+            return secret_json
+        except botocore.exceptions.ClientError as e:
+            print(
+                f"Could not get client secret '{secret_name}' from secrets manager")
+            raise e
+
+    def get_service_principal_token(self, resource_prefix):
+
+        token_url_ssm_path = f'/osdu/{resource_prefix}/oauth-token-uri'
+        aws_oauth_custom_scope_ssm_path = f'/osdu/{resource_prefix}/oauth-custom-scope'
+        client_id_ssm_path = f'/osdu/{resource_prefix}/client-credentials-client-id'
+        client_secret_name = f'/osdu/{resource_prefix}/client_credentials_secret'
+        client_secret_dict_key = 'client_credentials_client_secret'
+
+        client_id = self._get_ssm_parameter(client_id_ssm_path)
+        client_secret = self._get_secret(
+            client_secret_name, client_secret_dict_key)
+        token_url = self._get_ssm_parameter(token_url_ssm_path)
+        aws_oauth_custom_scope = self._get_ssm_parameter(
+            aws_oauth_custom_scope_ssm_path)
+
+        auth = '{}:{}'.format(client_id, client_secret)
+        encoded_auth = base64.b64encode(str.encode(auth))
+
+        headers = {}
+        headers['Authorization'] = 'Basic ' + encoded_auth.decode()
+        headers['Content-Type'] = 'application/x-www-form-urlencoded'
+
+        token_url = '{}?grant_type=client_credentials&client_id={}&scope={}'.format(
+            token_url, client_id, aws_oauth_custom_scope)
+
+        response = requests.post(url=token_url, headers=headers)
+
+        return json.loads(response.content.decode())['access_token']
diff --git a/osdu/client/aws.py b/osdu/client/aws.py
@@ -1,10 +1,11 @@
 import os
 import boto3
 
-from .base import BaseOsduClient
+from ._base import BaseOsduClient
 import boto3.session
 
 
+
 class AwsOsduClient(BaseOsduClient):
     """Good for batch tasks that don't have an interactive front-end. Token management is handled 
     with the boto3 library directly through the Cognito service. You have to supply additional arguments for this.