From 5db98b16131281f8a3dedd35073daf6271c5bef6 Mon Sep 17 00:00:00 2001 From: Tim Reichard Date: Thu, 19 Oct 2023 09:18:06 -0500 Subject: [PATCH 1/2] Add new DB_CONNECT class --- HISTORY.rst | 14 ++++++++ aioradio/ds_utils.py | 69 +++++++++++++++++++++++++++++++++++++++ aioradio/requirements.txt | 16 ++++----- setup.py | 5 ++- 4 files changed, 93 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 725e929..0aef824 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,20 @@ History ======= +v0.19.2 (2023-10-19) + +* Add new DB_CONNECT class that uses env vars instead of AWS secret manager for sensitive creds. +* Update aioboto3==11.3.1. +* Update cython==3.0.4. +* Update httpx==0.25.0. +* Update pandas==2.1.1. +* Update pre-commit==3.5.0. +* Update psycopg2-binary==2.9.9. +* Update pylint==3.0.1. +* Update pytest==7.4.2. +* Update + + v0.19.1 (2023-09-07) * Set logging level in ds_utils. diff --git a/aioradio/ds_utils.py b/aioradio/ds_utils.py index 4a86280..2de6693 100644 --- a/aioradio/ds_utils.py +++ b/aioradio/ds_utils.py @@ -268,6 +268,75 @@ def get_domino_connection(secret_id, project, host, env='sandbox'): return Domino(project=project, api_key=api_key, host=host) +class DB_CONNECT(): + """[Class for database connection] + + DB_CONNECT can be used to return a connection object, and will work + within the "with" context to appropriately commit or rollback + transactions based on the current environment. Uses env vars + instead of AWS secret manager for sensitive creds. + """ + config = None + conn = None + + def __init__(self, config: dict): + self.config = config + + def __enter__(self): + self.conn = self.connect() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.conn is not None: + if self.config.get('rollback', True): + self.conn.rollback() + else: + self.conn.commit() + self.conn.close() + + def connect(self): + """[Setup database connection.] + + Create and return a connection using the config info. Autocommit + will be on by default unless the rollback setting is set to true + in db_info Autocommit can be changed if needed once the + connection is returned + """ + + if self.config['db'] == 'psycopg2': + + from aioradio.psycopg2 import establish_psycopg2_connection + + self.conn = establish_psycopg2_connection( + host=self.config['host'], + user=self.config['user'], + password=self.config['password'], + database=self.config.get('database', ''), + port=self.config['port'] + ) + self.conn.autocommit = not self.config['rollback'] + + elif self.config['db'] == 'pyodbc': + + from aioradio.pyodbc import establish_pyodbc_connection + + self.conn = establish_pyodbc_connection( + host=self.config['host'], + user=self.config['user'], + pwd=self.config['pwd'], + port=self.config.get('port', None), + multi_subnet_failover=self.config.get('multi_subnet_failover', None), + database=self.config.get('database', ''), + driver=self.config.get('driver', '{ODBC Driver 17 for SQL Server}'), + autocommit = not self.config.get('rollback', True), + trusted_connection=self.config.get('trusted_connection', 'no'), + tds_version=self.config.get('tds_version', '7.4'), + application_intent=self.config.get('application_intent', '') + ) + + return self.conn + + class DbInfo(): """[Class for database connection] diff --git a/aioradio/requirements.txt b/aioradio/requirements.txt index 86f4bb9..0556a83 100644 --- a/aioradio/requirements.txt +++ b/aioradio/requirements.txt @@ -1,27 +1,27 @@ -aioboto3==11.3.0 +aioboto3==11.3.1 aiojobs==1.2.0 backoff==2.2.1 boto3==1.28.17 botocore==1.31.17 -cython==3.0.2 +cython==3.0.4 ddtrace==1.11.2 dominodatalab==1.2.4 fakeredis==1.10.1 faust-cchardet==2.1.19 flask==2.1.2 flask-cors==3.0.10 -httpx==0.24.0 +httpx==0.25.0 mandrill==1.0.60 moto==3.1.18 openpyxl==3.0.10 orjson==3.8.10 -pandas==2.1.0 -pre-commit==3.3.3 -psycopg2-binary==2.9.7 -pylint==2.17.5 +pandas==2.1.1 +pre-commit==3.5.0 +psycopg2-binary==2.9.9 +pylint==3.0.1 pyodbc==4.0.39 --no-binary=pyodbc pysmb==1.2.9.1 -pytest==7.4.0 +pytest==7.4.2 pytest-asyncio==0.21.1 pytest-cov==4.1.0 python-json-logger==2.0.7 diff --git a/setup.py b/setup.py index 20cefd9..3d7f23e 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ long_description = fileobj.read() setup(name='aioradio', - version='0.19.1', + version='0.19.2', description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more', long_description=long_description, long_description_content_type="text/markdown", @@ -21,7 +21,7 @@ ], install_requires=[ 'cython>=0.29.33', - 'aioboto3==11.3.0', + 'aioboto3==11.3.1', 'aiojobs>=1.0.0', 'backoff>=2.1.2', 'botocore==1.31.17', @@ -35,7 +35,6 @@ 'openpyxl==3.0.10', 'orjson>=3.6.8', 'pandas>=1.3.5', - #'psycopg2-binary>=2.9.3', 'pysmb>=1.2.7', 'python-json-logger>=2.0.2', 'redis==3.5.3' From 988c8d054959a884b5305d00fb7776a2edd62956 Mon Sep 17 00:00:00 2001 From: Tim Reichard Date: Thu, 19 Oct 2023 09:18:32 -0500 Subject: [PATCH 2/2] remove extra comment in history under v0.19.2 --- HISTORY.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0aef824..dbe5f1d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,7 +14,6 @@ v0.19.2 (2023-10-19) * Update psycopg2-binary==2.9.9. * Update pylint==3.0.1. * Update pytest==7.4.2. -* Update v0.19.1 (2023-09-07)