From 9dfddd2dc9a214daa097d2ee3cd2f803a0fd32aa Mon Sep 17 00:00:00 2001 From: Mike Duffy Date: Fri, 25 Feb 2022 10:46:51 -0600 Subject: [PATCH] Feature: AwsServicePrincipalOsduClient (#36) * Added: AwsServicePrincipalOsduClient * Added: _service_prinicpal_util.py from AWS with revisions documented in module comments. * Added: unit test for AwsServicePrincipalOsduClient * Updated client/__init__.py to import all of the concrete client classes into the top-level package namespace for cleaner importing. * Renamed: client/base.py --> _base.py as it should not be imported from outside of package. * README updates * Added: integration tests * Updated version --- README.md | 77 +++++++++++----- VERSION | 2 +- osdu/client/__init__.py | 4 +- osdu/client/{base.py => _base.py} | 9 +- osdu/client/_service_principal_util.py | 119 +++++++++++++++++++++++++ osdu/client/aws.py | 3 +- osdu/client/aws_service_principal.py | 15 ++++ osdu/client/simple.py | 2 +- tests/integration.py | 46 ++++++++-- tests/unit.py | 79 ++++++++++++---- 10 files changed, 297 insertions(+), 59 deletions(-) rename osdu/client/{base.py => _base.py} (87%) create mode 100644 osdu/client/_service_principal_util.py create mode 100644 osdu/client/aws_service_principal.py diff --git a/README.md b/README.md index db36333..c539a2f 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,14 @@ A simple python client for the [OSDU](https://community.opengroup.org/osdu) data - [Clients](#clients) - [SimpleOsduClient](#simpleosduclient) + - [AwsServicePrincipalOsduClient](#awsserviceprincipalosduclient) - [AwsOsduClient](#awsosduclient) - [Currently supported methods](#currently-supported-methods) - [Installation](#installation) - [Tests](#tests) - [Usage](#usage) - [Instantiating the SimpleOsduClient](#instantiating-the-simpleosduclient) + - [Instantiating the AwsServicePrincipalOsduClient](#instantiating-the-awsosduclient) - [Instantiating the AwsOsduClient](#instantiating-the-awsosduclient) - [Using the client](#using-the-client) - [Search for records by query](#search-for-records-by-query) @@ -37,6 +39,15 @@ login form or otheer mechanism. With this SimpleOsduClient, you simply provide t With this simplicity, you are also then respnsible for reefreeshing the token as needed and re-instantiating the client with the new token. +### AwsOsduServicePrincipalClient + +**Requires**: `boto3==1.15.*` + +Good for batch tasks that don't have an interactive front-end. Token management is handled +with the boto3 library directly through the Cognito service. You have to supply additional arguments for this. + +For OSDU on AWS, this client is usually simpler than the AwsOsduClient as long as you have IAM credentials to access the necessary resources. You only need to provide the OSDU resource_prefix, region, and profile. + ### AwsOsduClient **Requires**: `boto3==1.15.*` @@ -44,12 +55,14 @@ re-instantiating the client with the new token. Good for batch tasks that don't have an interactive front-end. Token management is handled with the boto3 library directly through the Cognito service. You have to supply additional arguments for this. +For OSDU on AWS, this client is useful in the case where you may want to perform actions as a specific OSDU user rather than as the ServicePrinicpal. + ## Currently supported methods -- [search](osdu/search.py) +- [search](osdu/services/search.py) - query - query_with_paging -- [storage](osdu/storage.py) +- [storage](osdu/services/storage.py) - query_all_kinds - get_record - get_records @@ -58,13 +71,13 @@ with the boto3 library directly through the Cognito service. You have to supply - store_records - delete_record - purge_record -- [dataset](osdu/dataset.py) +- [dataset](osdu/services/dataset.py) - get_dataset_registry - get_dataset_registries - get_storage_instructions - register_dataset - get_retrieval_instructions -- [entitlement](osdu/entitlement.py) +- [entitlements](osdu/services/entitlements.py) - get_groups - get_group_members - add_group_member @@ -98,18 +111,34 @@ python -m unittest -v tests.integration If environment variable `OSDU_API_URL` is set, then it does not need to be passed as an argument. Otherwise it must be passed as keyword argument. ```python -from osdu.client.simple import SimpleOsduClient +from osdu.client import SimpleOsduClient -data_partition = 'opendes' +data_partition = 'osdu' token = 'token-received-from-front-end-app' # With env var `OSDU_API_URL` set in current environment. -osdu = SimpleOsduClient(data_partition, token) +osdu_client = SimpleOsduClient(data_partition, token) # Without env var set. api_url = 'https://your.api.base_url.com' -osdu = SimpleOsduClient(data_partition, token, api_url=api_url) +osdu_client = SimpleOsduClient(data_partition, token, api_url=api_url) + +``` + +### Instantiating the AwsServicePrincipalOsduClient +```python +from osdu.client import AwsOsduClient + +data_partition = 'osdu' +resource_prefix = 'osdur3mX' + +osdu_client = AwsServicePrincipalOsduClient( + data_partition, + resource_prefix, + profile=os.environ['AWS_PROFILE'], + region=os.environ['AWS_DEFAULT_REGION'] +) ``` ### Instantiating the AwsOsduClient @@ -125,18 +154,18 @@ Environment variables: 1. `AWS_SECRETHASH` ```python -from osdu.client.aws import AwsOsduClient +from osdu.client import AwsOsduClient data_partition = 'osdu' -osdu = AwsOsduClient(data_partition) +osdu_client = AwsOsduClient(data_partition) ``` If you have not set the above environment variales—or you have only set some—then you will need to pass any undefined as args when instantiating the client. ```python from getpass import getpass -from osdu.client.aws import AwsOsduClient +from osdu.client import AwsOsduClient api_url = 'https://your.api.url.com' # Must be base URL only client_id = 'YOURCLIENTID' @@ -152,7 +181,7 @@ secretHash = base64.b64encode(dig).decode() -osdu = AwsOsduClient(data_partition, +osdu_client = AwsOsduClient(data_partition, api_url=api_url, client_id=client_id, user=user, @@ -169,9 +198,9 @@ Below are just a few usage examples. See [integration tests](https://github.com/ ```python query = { - "kind": f"opendes:osdu:*:*" + "kind": f"osdu:wks:*:*" } -result = osdu.search.query(query) +result = osdu_client.search.query(query) # { results: [ {...}, .... ], totalCount: ##### } ``` @@ -182,10 +211,10 @@ For result sets larger than 1,000 records, use the query with paging method. ```python page_size = 100 # Number of records per page (1-1000) query = { - "kind": f"opendes:osdu:*:*", + "kind": f"osdu:wks:*:*", "limit": page_size } -result = osdu.search.query_with_paging(query) +result = osdu_client.search.query_with_paging(query) # Iterate over the pages to do something with the results. for page, total_count in result: @@ -197,7 +226,7 @@ for page, total_count in result: ```python record_id = 'opendes:doc:123456789' -result = osdu.storage.get_record(record_id) +result = osdu_client.storage.get_record(record_id) # { 'id': 'opendes:doc:123456789', 'kind': ..., 'data': {...}, 'acl': {...}, .... } ``` @@ -208,14 +237,14 @@ new_or_updated_record = './record-123.json' with open(new_or_updated_record, 'r') as _file: record = json.load(_file) -result = osdu.storage.store_records([record]) +result = osdu_client.storage.store_records([record]) ``` #### List groupmembership for the current user ```python -result = osduClient.entitlements.get_groups() +result = osdu_client.entitlements.get_groups() # { # "desId": "user@example.org", # "groups": [ @@ -243,7 +272,7 @@ result = osduClient.entitlements.get_groups() ### List membership of a particular group ```python -result = osduClient.entitlements.get_group_members('users@osdu.example.com') +result = osdu_client.entitlements.get_group_members('users@osdu.example.com') #{ # "members": [ # { @@ -272,13 +301,13 @@ query = { #OWNER or MEMBER "role": "MEMBER", } -result = osduClient.entitlements.add_group_member('users.datalake.viewers@osdu.example.com',query) +result = osdu_client.entitlements.add_group_member('users.datalake.viewers@osdu.example.com',query) query = { "email": "user@example.com", #OWNER or MEMBER "role": "OWNER", } -result = osduClient.entitlements.add_group_member('service.search.admin@osdu.example.com',query) +result = osdu_client.entitlements.add_group_member('service.search.admin@osdu.example.com',query) ``` ### Delete user from a particular group @@ -291,11 +320,11 @@ query = { #OWNER or MEMBER "role": "MEMBER", } -result = osduClient.entitlements.delete_group_member('users.datalake.viewers@osdu.example.com',query) +result = osdu_client.entitlements.delete_group_member('users.datalake.viewers@osdu.example.com',query) query = { "email": "user@example.com", #OWNER or MEMBER "role": "OWNER", } -result = osduClient.entitlements.delete_group_member('service.search.admin@osdu.example.com',query) +result = osdu_client.entitlements.delete_group_member('service.search.admin@osdu.example.com',query) ``` diff --git a/VERSION b/VERSION index 0d91a54..1d0ba9e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 +0.4.0 diff --git a/osdu/client/__init__.py b/osdu/client/__init__.py index 1eb216d..08245a3 100644 --- a/osdu/client/__init__.py +++ b/osdu/client/__init__.py @@ -1 +1,3 @@ -__all__ = ['aws', 'simple'] \ No newline at end of file +from osdu.client.aws import AwsOsduClient +from osdu.client.simple import SimpleOsduClient +from osdu.client.aws_service_principal import AwsServicePrincipalOsduClient \ No newline at end of file diff --git a/osdu/client/base.py b/osdu/client/_base.py similarity index 87% rename from osdu/client/base.py rename to osdu/client/_base.py index 9e9f7df..fd613c8 100644 --- a/osdu/client/base.py +++ b/osdu/client/_base.py @@ -54,7 +54,10 @@ def __init__(self, data_partition_id, api_url: str = None): """ self._data_partition_id = data_partition_id # TODO: Validate api_url against URL regex pattern. - self._api_url = (api_url or os.environ.get('OSDU_API_URL')).rstrip('/') + api_url = api_url or os.environ.get('OSDU_API_URL') + if not api_url: + raise Exception('No API URL found.') + self._api_url = api_url.rstrip('/') # Instantiate services. self._search = SearchService(self) @@ -64,7 +67,3 @@ def __init__(self, data_partition_id, api_url: str = None): # TODO: Implement these services. # self.__legal = LegaService(self) - # Abstract Method - def get_tokens(self, password): - raise NotImplementedError( - 'This method must be implemented by a subclass') diff --git a/osdu/client/_service_principal_util.py b/osdu/client/_service_principal_util.py new file mode 100644 index 0000000..778ee85 --- /dev/null +++ b/osdu/client/_service_principal_util.py @@ -0,0 +1,119 @@ +# Copyright © 2020 Amazon Web Services +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# =================================== +# REVISIONS +# =================================== +# +# Date Author & Description +# --------- --------------------- +# 2022-02-23 mike.duffy@parivedasolutions.com +# - Updated constructor to optionally accept AWS profile and region instead of +# a boto3 session. +# - Constructor to require resource_prefix and make this an instance variable. +# - Refactored _get_secret method to fix UnboundLocalError for local variable 'secret'. +# - Refactored _get_secret method to simplify try/except flow and to print secret_name on exception. +# - Updated formatting to be PEP8-compliant. +# +import base64 +import boto3 +import requests +import json +import botocore.exceptions + + +class ServicePrincipalUtil: + + @property + def api_url(self): + return self._api_url + + def __init__( + self, + resource_prefix: str, + aws_session: boto3.Session = None, + region: str = None, + profile: str = None + ): + """If a session is not provided, then region and profile must be provided. + If none of these are provided, then boto3.Session will check for env vars: AWS_PROFILE and AWS_DEFAULT_REGION. + If not found there, then instantiation will fail. + + :param resource_prefix: Resource prefix from OSDU deployment. e.g. 'osdur3mX' + :param aws_session: boto3 sesssion to use for retrieving paramaeters an secrets for the OSDU instance. + :param region: AWS Region where OSDU instance is deployed. e.g. 'us-east-1' + :param profile: AWS credentials (CLI) profile name. + """ + # If a boto session is provided, then use it. Otherwise, instantiate a new one with provided + # region and profile. + if aws_session: + self._session = aws_session + else: + self._session = boto3.Session( + region_name=region, profile_name=profile) + self._api_url = self._get_ssm_parameter( + f'/osdu/{resource_prefix}/api/url') + + def _get_ssm_parameter(self, ssm_path): + ssm_client = self._session.client('ssm') + ssm_response = ssm_client.get_parameter(Name=ssm_path) + return ssm_response['Parameter']['Value'] + + def _get_secret(self, secret_name, secret_dict_key): + client = self._session.client(service_name='secretsmanager') + # In this sample we only handle the specific exceptions for the 'GetSecretValue' API. + # See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html + try: + secret_response = client.get_secret_value(SecretId=secret_name) + secret_val = None + if 'SecretString' in secret_response: + secret_val = secret_response['SecretString'] + elif 'SecretBinary' in secret_response: + secret_val = base64.b64decode(secret_response['SecretBinary']) + secret_json = json.loads(secret_val)[secret_dict_key] + return secret_json + except botocore.exceptions.ClientError as e: + print( + f"Could not get client secret '{secret_name}' from secrets manager") + raise e + + def get_service_principal_token(self, resource_prefix): + + token_url_ssm_path = f'/osdu/{resource_prefix}/oauth-token-uri' + aws_oauth_custom_scope_ssm_path = f'/osdu/{resource_prefix}/oauth-custom-scope' + client_id_ssm_path = f'/osdu/{resource_prefix}/client-credentials-client-id' + client_secret_name = f'/osdu/{resource_prefix}/client_credentials_secret' + client_secret_dict_key = 'client_credentials_client_secret' + + client_id = self._get_ssm_parameter(client_id_ssm_path) + client_secret = self._get_secret( + client_secret_name, client_secret_dict_key) + token_url = self._get_ssm_parameter(token_url_ssm_path) + aws_oauth_custom_scope = self._get_ssm_parameter( + aws_oauth_custom_scope_ssm_path) + + auth = '{}:{}'.format(client_id, client_secret) + encoded_auth = base64.b64encode(str.encode(auth)) + + headers = {} + headers['Authorization'] = 'Basic ' + encoded_auth.decode() + headers['Content-Type'] = 'application/x-www-form-urlencoded' + + token_url = '{}?grant_type=client_credentials&client_id={}&scope={}'.format( + token_url, client_id, aws_oauth_custom_scope) + + response = requests.post(url=token_url, headers=headers) + + return json.loads(response.content.decode())['access_token'] diff --git a/osdu/client/aws.py b/osdu/client/aws.py index 6047849..266b7d1 100644 --- a/osdu/client/aws.py +++ b/osdu/client/aws.py @@ -1,10 +1,11 @@ import os import boto3 -from .base import BaseOsduClient +from ._base import BaseOsduClient import boto3.session + class AwsOsduClient(BaseOsduClient): """Good for batch tasks that don't have an interactive front-end. Token management is handled with the boto3 library directly through the Cognito service. You have to supply additional arguments for this. diff --git a/osdu/client/aws_service_principal.py b/osdu/client/aws_service_principal.py new file mode 100644 index 0000000..4d12f67 --- /dev/null +++ b/osdu/client/aws_service_principal.py @@ -0,0 +1,15 @@ +from ._base import BaseOsduClient +from ._service_principal_util import ServicePrincipalUtil + + +class AwsServicePrincipalOsduClient(BaseOsduClient): + + def __init__(self, data_partition_id: str, resource_prefix: str, profile: str = None, region: str = None): + self._sp_util = ServicePrincipalUtil( + resource_prefix, profile=profile, region=region) + self._resource_prefix = resource_prefix + self._access_token = self._get_tokens() + super().__init__(data_partition_id, self._sp_util.api_url) + + def _get_tokens(self): + return self._sp_util.get_service_principal_token(self._resource_prefix) diff --git a/osdu/client/simple.py b/osdu/client/simple.py index b88d39a..70cfe92 100644 --- a/osdu/client/simple.py +++ b/osdu/client/simple.py @@ -1,4 +1,4 @@ -from .base import BaseOsduClient +from ._base import BaseOsduClient class SimpleOsduClient(BaseOsduClient): diff --git a/tests/integration.py b/tests/integration.py index b69c4f1..9d7c221 100644 --- a/tests/integration.py +++ b/tests/integration.py @@ -4,14 +4,16 @@ switch between OSDU environments. """ import json -from logging import exception +import os from unittest import TestCase -from dotenv import load_dotenv import requests -from osdu.client.aws import AwsOsduClient -from osdu.client.simple import SimpleOsduClient - +from dotenv import load_dotenv +from osdu.client import ( + AwsOsduClient, + AwsServicePrincipalOsduClient, + SimpleOsduClient +) load_dotenv(verbose=True, override=True) @@ -21,7 +23,6 @@ class TestSimpleOsduClient(TestCase): def test_endpoint_access(self): - # token = os.environ.get('OSDU_ACCESS_TOKEN') token = AwsOsduClient(data_partition).access_token query = { "kind": f"*:*:*:*", @@ -41,6 +42,35 @@ def test_get_access_token(self): self.assertIsNotNone(client.access_token) +class TestAwsServicePrincipalOsduClient(TestCase): + + def test_get_access_token(self): + client = AwsServicePrincipalOsduClient( + data_partition, + os.environ['OSDU_RESOURCE_PREFIX'], + profile=os.environ['AWS_PROFILE'], + region=os.environ['AWS_DEFAULT_REGION'] + ) + self.assertIsNotNone(client.access_token) + self.assertIsNotNone(client.api_url) + + def test_endpoint_access(self): + query = { + "kind": f"*:*:*:*", + "limit": 1 + } + client = AwsServicePrincipalOsduClient( + data_partition, + os.environ['OSDU_RESOURCE_PREFIX'], + profile=os.environ['AWS_PROFILE'], + region=os.environ['AWS_DEFAULT_REGION'] + ) + + result = client.search.query(query)['results'] + + self.assertEqual(1, len(result)) + + class TestOsduServiceBase(TestCase): @classmethod @@ -107,7 +137,7 @@ def test_basic_paging(self): # Iterate over first 'max_pages' pages and check that each page contains 'page_size' results. page_count = 1 - for page, total_count in result: + for page, _ in result: with (self.subTest(i=page_count)): self.assertEqual(page_size, len( page), f'Failed on page #{page_count}') @@ -228,7 +258,7 @@ def test_002_delete_record_only_soft_deletes(self): record_id)).get('recordId') == record_id with self.assertRaises(requests.RequestException) as context: self.osdu.storage.get_record(record_id) # Should throw exception - + # Assert self.assertTrue(record_was_deleted) self.assertTrue(record_still_has_versions) diff --git a/tests/unit.py b/tests/unit.py index 3d5d8c8..1e4c77b 100644 --- a/tests/unit.py +++ b/tests/unit.py @@ -1,39 +1,82 @@ +import base64 +import hashlib +import hmac from unittest import TestCase, mock -from osdu.client.aws import AwsOsduClient -from osdu.client.simple import SimpleOsduClient +from osdu.client import ( + AwsOsduClient, + AwsServicePrincipalOsduClient, + SimpleOsduClient +) -import hmac -import hashlib -import base64 + +class TestAwsServicePrincipalOsduClient(TestCase): + + @mock.patch('osdu.client._service_principal_util.ServicePrincipalUtil.get_service_principal_token') + @mock.patch('boto3.Session') + @mock.patch('base64.b64decode') + def test_initialize_aws_client_with_args(self, mock_b64decode, mock_session, mock_sputil): + partition = 'osdu' + resource_prefix = 'r3mx' + region = 'us-east-1' + profile = 'myprofile' + + client = AwsServicePrincipalOsduClient( + partition, resource_prefix, profile=profile, region=region) + + self.assertIsNotNone(client) + self.assertEqual(partition, client.data_partition_id) + self.assertIsNotNone(client.access_token) + self.assertIsNotNone(client.api_url) class TestAwsOsduClient(TestCase): @mock.patch('boto3.Session') - def test_initialize_aws_client_with_args(self, mock_session): + def test_initialize_aws_client(self, mock_session): partition = 'opendes' api_url = 'https://your.api.url.com' client_id = 'YOURCLIENTID' - client_secret = 'YOURCLIENTSECRET' user = 'username@testing.com' password = 'p@ssw0rd' profile = 'osdu-dev' + client = AwsOsduClient( + partition, + api_url=api_url, + client_id=client_id, + user=user, + password=password, + profile=profile + ) + self.assertIsNotNone(client) + self.assertEqual(partition, client.data_partition_id) - message = user + client_id - dig = hmac.new(client_secret.encode('UTF-8'), msg=message.encode('UTF-8'),digestmod=hashlib.sha256).digest() - secretHash = base64.b64encode(dig).decode() - - + @mock.patch('boto3.Session') + def test_initialize_aws_client_with_client_secret(self, mock_session): + partition = 'opendes' + api_url = 'https://your.api.url.com' + client_id = 'YOURCLIENTID' + client_secret = 'YOURCLIENTSECRET' + user = 'username@testing.com' + password = 'p@ssw0rd' + profile = 'osdu-dev' - client = AwsOsduClient(partition, - api_url=api_url, - client_id=client_id, - user=user, - password=password, - profile=profile) + message = user + client_id + dig = hmac.new( + client_secret.encode('UTF-8'), msg=message.encode('UTF-8'), digestmod=hashlib.sha256).digest() + secret_hash = base64.b64encode(dig).decode() + + client = AwsOsduClient( + partition, + api_url=api_url, + client_id=client_id, + user=user, + password=password, + secret_hash=secret_hash, + profile=profile + ) self.assertIsNotNone(client) self.assertEqual(partition, client.data_partition_id)