Add option to pass aws_creds argument in get_secret function

HISTORY.rst

@@ -3,6 +3,13 @@ History
 =======
 
 
+v0.17.26 (2023-01-12)
+
+* Add option to pass aws_creds argument in get_secret function.
+* Update httpx==0.23.3.
+* Update orjson==3.8.5.
+
+
 v0.17.25 (2023-01-03)
 
 * Update httpx==0.23.2.

aioradio/aws/secrets.py

@@ -1,41 +1,27 @@
 """Generic async AWS functions for Secrets Manager."""
 
-import base64
-from typing import List
+from base64 import b64decode
+from typing import Dict
 
-from aioradio.aws.utils import AwsServiceManager
+import boto3
 
-AWS_SERVICE = AwsServiceManager(service='secretsmanager', regions=['us-east-1'])
-SECRETS = AWS_SERVICE.service_dict
 
-
-async def add_regions(regions: List[str]):
-    """Add regions to Secret Manager AWS service.
-
-    Args:
-        regions (List[str]): List of AWS regions
-    """
-
-    AWS_SERVICE.add_regions(regions)
-
-
-@AWS_SERVICE.active
-async def get_secret(secret_name: str, region: str) -> str:
+async def get_secret(secret_name: str, region: str, aws_creds: Dict[str, str]=None) -> str:
     """Get secret from AWS Secrets Manager.
 
     Args:
         secret_name (str): secret name
         region (str): AWS region
+        aws_creds (Dict[str, str], optional): AWS credentials
 
     Returns:
         str: secret value
     """
 
-    secret = ''
-    response = await SECRETS[region]['client']['obj'].get_secret_value(SecretId=secret_name)
-    if 'SecretString' in response:
-        secret = response['SecretString']
+    if aws_creds:
+        client = boto3.client(service_name='secretsmanager', region_name=region, **aws_creds)
     else:
-        secret = base64.b64decode(response['SecretBinary'])
+        client = boto3.client(service_name='secretsmanager', region_name=region)
 
-    return secret
+    resp = client.get_secret_value(SecretId=secret_name)
+    return resp['SecretString'] if 'SecretString' in resp else b64decode(resp['SecretBinary'])

aioradio/file_ingestion.py

@@ -1303,7 +1303,11 @@ async def child_wrapper(*args, **kwargs) -> Any:
             for item in db_info:
 
                 if item['db'] in ['pyodbc', 'psycopg2']:
-                    creds = {**json.loads(await get_secret(item['secret'], item['region'])), **{'database': item.get('database', '')}}
+                    if 'aws_creds' in item:
+                        secret = await get_secret(item['secret'], item['region'], item['aws_creds'])
+                    else:
+                        secret = await get_secret(item['secret'], item['region'])
+                    creds = {**json.loads(secret), **{'database': item.get('database', '')}}
                     if item['db'] == 'pyodbc':
                         # Add import here because it requires extra dependencies many systems
                         # don't have out of the box so only import when explicitly being used

aioradio/requirements.txt

@@ -9,11 +9,11 @@ dominodatalab==1.2.1
 fakeredis==1.10.1
 flask==2.1.2
 flask-cors==3.0.10
-httpx==0.23.2
+httpx==0.23.3
 mandrill==1.0.60
 moto==3.1.18
 openpyxl==3.0.10
-orjson==3.8.3
+orjson==3.8.5
 pandas==1.4.4
 pre-commit==2.21.0
 psycopg2-binary==2.9.5

aioradio/tests/aws_secrets_test.py

@@ -2,23 +2,27 @@
 
 # pylint: disable=unused-argument
 
+import asyncio
+
+import boto3
 import pytest
+from moto import mock_secretsmanager
 
-from aioradio.aws.secrets import add_regions, get_secret
+from aioradio.aws.secrets import get_secret
 
 pytestmark = pytest.mark.asyncio
 
 
-async def test_add_regions():
-    """Add us-east-2 region."""
-
-    await add_regions(['us-east-2'])
-
-
-async def test_secrets_get_secret(create_secret):
+@mock_secretsmanager
+def test_secrets_get_secret():
     """Test getting secret from Secrets Manager."""
 
-    secret = await get_secret(secret_name='test-secret', region='us-east-2')
+    client = boto3.client('secretsmanager', region_name='us-east-1')
+    result = client.create_secret(Name="test-secret-aioradio", SecretString="abc123")
+    assert result["ARN"]
+
+    loop = asyncio.get_event_loop()
+    secret = loop.run_until_complete(get_secret(secret_name='test-secret-aioradio', region='us-east-1'))
     assert secret == 'abc123'
 
 

conftest.py

@@ -6,14 +6,14 @@
 
 import aioboto3
 import aiobotocore
+import boto3
 import pytest
 import pytest_asyncio
 from aiobotocore.config import AioConfig
 
 from aioradio.aws.dynamodb import DYNAMO
 from aioradio.aws.moto_server import MotoService
 from aioradio.aws.s3 import S3
-from aioradio.aws.secrets import SECRETS
 from aioradio.aws.sqs import SQS
 from aioradio.redis import Redis
 
@@ -218,35 +218,6 @@ async def delete_sqs_queue(sqs_client, queue_url):
     assert_status_code(response, 200)
 
 
-######### aiobotocore secretsmanager async moto fixtures #########
-
-@pytest.fixture(scope='module')
-def secrets_manager_config(region):
-    return AioConfig(region_name=region, read_timeout=5, connect_timeout=5)
-
-
-@pytest_asyncio.fixture(scope='module')
-async def secrets_manager_server():
-    async with MotoService('secretsmanager', port=5003) as svc:
-        yield svc.endpoint_url
-
-
-@pytest_asyncio.fixture(scope='module')
-async def secrets_manager_client(session, region, secrets_manager_config, secrets_manager_server):
-    kw = moto_config(secrets_manager_server)
-    async with session.create_client('secretsmanager', region_name=region, config=secrets_manager_config, **kw) as client:
-        real_client = SECRETS[region]['client']
-        SECRETS[region]['client']['obj'] = client
-        yield client
-        SECRETS[region]['client'] = real_client
-
-
-@pytest_asyncio.fixture(scope='module')
-async def create_secret(secrets_manager_client):
-    result = await secrets_manager_client.create_secret(Name="test-secret", SecretString="abc123")
-    assert result["ARN"]
-
-
 ######### aioboto3 dynamodb async moto fixtures  #########
 
 @pytest.fixture(scope='module')

setup.py

@@ -7,7 +7,7 @@
     long_description = fileobj.read()
 
 setup(name='aioradio',
-    version='0.17.25',
+    version='0.17.26',
     description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more',
     long_description=long_description,
     long_description_content_type="text/markdown",