From 02717bc6f24a6f5905dd5698954872a704189fbe Mon Sep 17 00:00:00 2001 From: "tim.reichard" Date: Tue, 18 Apr 2023 11:22:54 -0500 Subject: [PATCH] Add psycopg2 imports inside functions --- HISTORY.rst | 5 +++++ aioradio/ds_utils.py | 8 +++++--- aioradio/file_ingestion.py | 3 ++- aioradio/requirements.txt | 8 ++++---- setup.py | 2 +- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a02a33f..ca890c6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,11 @@ History ======= +v0.18.2 (2023-04-18) + +* Add psycopg2 imports inside functions and not at the top of the file. + + v0.18.1 (2023-04-13) * Remove psycopg2 as a required install with aioradio. diff --git a/aioradio/ds_utils.py b/aioradio/ds_utils.py index 1a970f2..26e0526 100644 --- a/aioradio/ds_utils.py +++ b/aioradio/ds_utils.py @@ -1,5 +1,6 @@ """utils.py.""" +# pylint: disable=import-outside-toplevel # pylint: disable=invalid-name # pylint: disable=logging-fstring-interpolation # pylint: disable=no-member @@ -24,9 +25,6 @@ from domino import Domino from smb.SMBConnection import SMBConnection -from aioradio.psycopg2 import establish_psycopg2_connection -from aioradio.pyodbc import establish_pyodbc_connection, pyodbc_query_fetchall - warnings.simplefilter(action='ignore', category=UserWarning) logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -81,6 +79,8 @@ def delete_s3_object(s3_client, bucket, s3_prefix): def get_fice_institutions_map(db_config): """Get mapping of fice to college from mssql table.""" + from aioradio.pyodbc import pyodbc_query_fetchall + result = {} with DbInfo(db_config) as target_db: query = "SELECT FICE, Institution FROM EESFileuploadAssignments WHERE FileCategory = 'EnrollmentLens'" @@ -298,6 +298,7 @@ def connect(self): secret_client = get_boto3_session(self.config['env']).client("secretsmanager", region_name='us-east-1') creds = json.loads(secret_client.get_secret_value(SecretId=self.config['secret'])['SecretString']) if self.config['db'] == 'psycopg2': + from aioradio.psycopg2 import establish_psycopg2_connection self.conn = establish_psycopg2_connection( host = creds['host'], user = creds['user'], @@ -307,6 +308,7 @@ def connect(self): ) self.conn.autocommit = not self.config['rollback'] elif self.config['db'] == 'pyodbc': + from aioradio.pyodbc import establish_pyodbc_connection self.conn = establish_pyodbc_connection( host = creds['host'], user = creds['user'], diff --git a/aioradio/file_ingestion.py b/aioradio/file_ingestion.py index f2be6ef..f0108da 100644 --- a/aioradio/file_ingestion.py +++ b/aioradio/file_ingestion.py @@ -39,7 +39,6 @@ from aioradio.aws.s3 import download_file, upload_file from aioradio.aws.secrets import get_secret -from aioradio.psycopg2 import establish_psycopg2_connection DIRECTORY = Path(__file__).parent.absolute() LOG = logging.getLogger('file_ingestion') @@ -122,6 +121,8 @@ async def child_wrapper(*args, **kwargs) -> Any: from aioradio.pyodbc import establish_pyodbc_connection conns[item['name']] = establish_pyodbc_connection(**creds, autocommit=False) elif item['db'] == 'psycopg2': + from aioradio.psycopg2 import \ + establish_psycopg2_connection conns[item['name']] = establish_psycopg2_connection(**creds) rollback[item['name']] = item['rollback'] print(f"ESTABLISHED CONNECTION for {item['name']}") diff --git a/aioradio/requirements.txt b/aioradio/requirements.txt index 1502deb..46cbc0c 100644 --- a/aioradio/requirements.txt +++ b/aioradio/requirements.txt @@ -4,7 +4,7 @@ backoff==2.2.1 boto3==1.24.59 botocore==1.27.59 cchardet==2.1.7 -ddtrace==1.9.4 +ddtrace==1.11.2 dominodatalab==1.2.3 fakeredis==1.10.1 flask==2.1.2 @@ -14,13 +14,13 @@ mandrill==1.0.60 moto==3.1.18 openpyxl==3.0.10 orjson==3.8.10 -pandas==1.5.3 +pandas==2.0.0 pre-commit==3.2.2 psycopg2-binary==2.9.6 pylint==2.17.2 -pyodbc==4.0.38 +pyodbc==4.0.39 pysmb==1.2.9.1 -pytest==7.3.0 +pytest==7.3.1 pytest-asyncio==0.21.0 pytest-cov==4.0.0 python-json-logger==2.0.7 diff --git a/setup.py b/setup.py index 6a4087e..30dd5cc 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ long_description = fileobj.read() setup(name='aioradio', - version='0.18.1', + version='0.18.2', description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more', long_description=long_description, long_description_content_type="text/markdown",