Add psycopg2 imports inside functions

HISTORY.rst

@@ -3,6 +3,11 @@ History
 =======
 
 
+v0.18.2 (2023-04-18)
+
+* Add psycopg2 imports inside functions and not at the top of the file.
+
+
 v0.18.1 (2023-04-13)
 
 * Remove psycopg2 as a required install with aioradio.

aioradio/ds_utils.py

@@ -1,5 +1,6 @@
 """utils.py."""
 
+# pylint: disable=import-outside-toplevel
 # pylint: disable=invalid-name
 # pylint: disable=logging-fstring-interpolation
 # pylint: disable=no-member
@@ -24,9 +25,6 @@
 from domino import Domino
 from smb.SMBConnection import SMBConnection
 
-from aioradio.psycopg2 import establish_psycopg2_connection
-from aioradio.pyodbc import establish_pyodbc_connection, pyodbc_query_fetchall
-
 warnings.simplefilter(action='ignore', category=UserWarning)
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -81,6 +79,8 @@ def delete_s3_object(s3_client, bucket, s3_prefix):
 def get_fice_institutions_map(db_config):
     """Get mapping of fice to college from mssql table."""
 
+    from aioradio.pyodbc import pyodbc_query_fetchall
+
     result = {}
     with DbInfo(db_config) as target_db:
         query = "SELECT FICE, Institution FROM EESFileuploadAssignments WHERE FileCategory = 'EnrollmentLens'"
@@ -298,6 +298,7 @@ def connect(self):
         secret_client = get_boto3_session(self.config['env']).client("secretsmanager", region_name='us-east-1')
         creds = json.loads(secret_client.get_secret_value(SecretId=self.config['secret'])['SecretString'])
         if self.config['db'] == 'psycopg2':
+            from aioradio.psycopg2 import establish_psycopg2_connection
             self.conn = establish_psycopg2_connection(
                 host = creds['host'],
                 user = creds['user'],
@@ -307,6 +308,7 @@ def connect(self):
             )
             self.conn.autocommit = not self.config['rollback']
         elif self.config['db'] == 'pyodbc':
+            from aioradio.pyodbc import establish_pyodbc_connection
             self.conn = establish_pyodbc_connection(
                 host = creds['host'],
                 user = creds['user'],

aioradio/file_ingestion.py

@@ -39,7 +39,6 @@
 
 from aioradio.aws.s3 import download_file, upload_file
 from aioradio.aws.secrets import get_secret
-from aioradio.psycopg2 import establish_psycopg2_connection
 
 DIRECTORY = Path(__file__).parent.absolute()
 LOG = logging.getLogger('file_ingestion')
@@ -122,6 +121,8 @@ async def child_wrapper(*args, **kwargs) -> Any:
                         from aioradio.pyodbc import establish_pyodbc_connection
                         conns[item['name']] = establish_pyodbc_connection(**creds, autocommit=False)
                     elif item['db'] == 'psycopg2':
+                        from aioradio.psycopg2 import \
+                            establish_psycopg2_connection
                         conns[item['name']] = establish_psycopg2_connection(**creds)
                     rollback[item['name']] = item['rollback']
                     print(f"ESTABLISHED CONNECTION for {item['name']}")

aioradio/requirements.txt

@@ -4,7 +4,7 @@ backoff==2.2.1
 boto3==1.24.59
 botocore==1.27.59
 cchardet==2.1.7
-ddtrace==1.9.4
+ddtrace==1.11.2
 dominodatalab==1.2.3
 fakeredis==1.10.1
 flask==2.1.2
@@ -14,13 +14,13 @@ mandrill==1.0.60
 moto==3.1.18
 openpyxl==3.0.10
 orjson==3.8.10
-pandas==1.5.3
+pandas==2.0.0
 pre-commit==3.2.2
 psycopg2-binary==2.9.6
 pylint==2.17.2
-pyodbc==4.0.38
+pyodbc==4.0.39
 pysmb==1.2.9.1
-pytest==7.3.0
+pytest==7.3.1
 pytest-asyncio==0.21.0
 pytest-cov==4.0.0
 python-json-logger==2.0.7

setup.py

@@ -7,7 +7,7 @@
     long_description = fileobj.read()
 
 setup(name='aioradio',
-    version='0.18.1',
+    version='0.18.2',
     description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more',
     long_description=long_description,
     long_description_content_type="text/markdown",