diff --git a/HISTORY.rst b/HISTORY.rst index 473320b..b2d949b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,11 @@ History ======= +v0.17.2 (2022-02-18) + +* Adding converted async to sync functions to ds_utils.py. + + v0.17.1 (2022-02-17) * Adding a data science utils script that can be shared across project easily. diff --git a/aioradio/ds_utils.py b/aioradio/ds_utils.py index e9c09a4..af35091 100644 --- a/aioradio/ds_utils.py +++ b/aioradio/ds_utils.py @@ -5,8 +5,10 @@ # pylint: disable=no-member # pylint: disable=too-many-arguments # pylint: disable=too-many-boolean-expressions +# pylint: disable=unnecessary-comprehension # pylint: disable=unused-argument +import base64 import csv import json import logging @@ -22,8 +24,8 @@ from domino import Domino from smb.SMBConnection import SMBConnection -from aioradio.pyodbc import establish_pyodbc_connection from aioradio.psycopg2 import establish_psycopg2_connection +from aioradio.pyodbc import establish_pyodbc_connection, pyodbc_query_fetchall warnings.simplefilter(action='ignore', category=UserWarning) logging.basicConfig(level=logging.INFO) @@ -40,6 +42,49 @@ def file_to_s3(s3_client, local_filepath, s3_bucket, key): logger.info(f"Uploading s3://{s3_bucket}/{key} took {round(time() - start, 2)} seconds") +def get_secret(s3_client, secret_name): + """Get secret from AWS Secrets Manager.""" + + secret = '' + response = s3_client.get_secret_value(SecretId=secret_name) + secret = response['SecretString'] if 'SecretString' in response else base64.b64decode(response['SecretBinary']) + + return secret + + +def list_s3_objects(s3_client, s3_bucket, s3_prefix, with_attributes=False): + """List all objects within s3 folder.""" + + arr = [] + paginator = s3_client.get_paginator('list_objects') + for result in paginator.paginate(Bucket=s3_bucket, Prefix=s3_prefix): + for item in result.get('Contents', []): + if with_attributes: + arr.append(item) + else: + arr.append(item['Key']) + + return arr + + +def delete_s3_object(s3_client, bucket, s3_prefix): + """Delete object(s) from s3.""" + + return s3_client.delete_object(Bucket=bucket, Key=s3_prefix) + + +def get_fice_institutions_map(db_config): + """Get mapping of fice to college from mssql table.""" + + result = {} + with DbInfo(db_config) as target_db: + query = "SELECT FICE, Institution FROM EESFileuploadAssignments WHERE FileCategory = 'EnrollmentLens'" + rows = pyodbc_query_fetchall(conn=target_db.conn, query=query) + result = {fice: institution for fice, institution in rows} + + return result + + def bytes_to_s3(s3_client, s3_bucket, key, body): """Write data in bytes to s3.""" diff --git a/setup.py b/setup.py index 14ba916..26afcee 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ long_description = fileobj.read() setup(name='aioradio', - version='0.17.1', + version='0.17.2', description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more', long_description=long_description, long_description_content_type="text/markdown", @@ -24,6 +24,7 @@ 'aiojobs>=0.3.0', 'boto3==1.20.24', 'ddtrace>=0.58.2', + 'dominodatalab @ https://github.com/dominodatalab/python-domino/archive/refs/tags/1.0.6.zip', 'fakeredis>=1.7.1', 'httpx>=0.19.0', 'mandrill>=1.0.60',