Skip to content

Commit

Permalink
Modify pyobdc & psycopg2 functions to run sync
Browse files Browse the repository at this point in the history
  • Loading branch information
tim.reichard committed Feb 11, 2022
1 parent 47aff01 commit 331437f
Show file tree
Hide file tree
Showing 9 changed files with 57 additions and 45 deletions.
14 changes: 13 additions & 1 deletion HISTORY.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,18 @@ History
=======


v0.17.0 (2022-02-11)

* Modify pyobdc & psycopg2 functions to no longer be async
* Update aiobotocore==2.1.0.
* Update httpx==0.22.0.
* Update moto==3.0.3.
* Update numpy==1.22.2.
* Update orjson==3.6.6.
* Update twine==3.8.0.
* Update pre-commit==2.17.0.
* Update pytest==7.0.0.

v0.16.2 (2022-01-13)

* Update EL3 field lengths.
Expand All @@ -19,7 +31,7 @@ v0.16.0 (2022-01-11)
* Update ddtrace==0.57.0.
* Update moto==2.3.1.
* Update numpy==1.22.0.
* Update orjson==3.6.5
* Update orjson==3.6.5.
* Update pre-commit==2.16.0.
* Update psycopg2-binary==2.9.3.
* Update pylint==2.12.2.
Expand Down
23 changes: 12 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ Besides what is shown below in the examples, there is also support for SQS, Dyna
```python
import asyncio

from aioradio.aws.s3 import create_bucket
from aioradio.aws.s3 import delete_s3_object
from aioradio.aws.s3 import download_file
from aioradio.aws.s3 import get_object
from aioradio.aws.s3 import list_s3_objects
from aioradio.aws.s3 import upload_file
from aioradio.aws.s3 import (
create_bucket,
delete_s3_object,
download_file,
get_object,
list_s3_objects,
upload_file
)

async def main():
s3_bucket = 'aioradio'
Expand Down Expand Up @@ -58,15 +60,15 @@ from aioradio.pyodbc import establish_pyodbc_connection
from aioradio.pyodbc import pyodbc_query_fetchone
from aioradio.pyodbc import pyodbc_query_fetchall

async def main():
conn = await establish_pyodbc_connection(host='your-host', user='your-user', pwd='your-password')
def main():
conn = establish_pyodbc_connection(host='your-host', user='your-user', pwd='your-password')

query = "SELECT homeruns FROM MLB.dbo.LosAngelesAngels WHERE lastname = 'Trout' AND year = '2020'"
row = await pyodbc_query_fetchone(conn=conn, query=query)
row = pyodbc_query_fetchone(conn=conn, query=query)
print(row)

query = "SELECT homeruns FROM MLB.dbo.LosAngelesAngels WHERE lastname = 'Trout'"
rows = await pyodbc_query_fetchall(conn=conn, query=query)
rows = pyodbc_query_fetchall(conn=conn, query=query)
print(rows)


Expand Down Expand Up @@ -144,6 +146,5 @@ See also the list of [contributors](https://github.com/nrccua/aioradio/graphs/co

## ACKNOWLEDGEMENTS

* **Bryan Cusatis** - Architect contributing to aioradio.
* **Kyle Edwards** - Developer contributing to aioradio.
* **Pedro Artiga** - Developer contributing to aioradio.
2 changes: 1 addition & 1 deletion aioradio/aws/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ async def establish_client_resource(self, service_dict: Dict[str, Any], item: st
await service_dict['obj'].__aexit__(None, None, None)

if self.module == 'aiobotocore':
service_dict['session'] = aiobotocore.get_session()
service_dict['session'] = aiobotocore.session.get_session()
service_dict['obj'] = await service_dict['session'].create_client(**kwargs).__aenter__()
elif self.module == 'aioboto3':
service_dict['session'] = aioboto3.Session()
Expand Down
4 changes: 2 additions & 2 deletions aioradio/file_ingestion.py
Original file line number Diff line number Diff line change
Expand Up @@ -1300,9 +1300,9 @@ async def child_wrapper(*args, **kwargs) -> Any:
# Add import here because it requires extra dependencies many systems
# don't have out of the box so only import when explicitly being used
from aioradio.pyodbc import establish_pyodbc_connection
conns[item['name']] = await establish_pyodbc_connection(**creds, autocommit=False)
conns[item['name']] = establish_pyodbc_connection(**creds, autocommit=False)
elif item['db'] == 'psycopg2':
conns[item['name']] = await establish_psycopg2_connection(**creds)
conns[item['name']] = establish_psycopg2_connection(**creds)
rollback[item['name']] = item['rollback']
print(f"ESTABLISHED CONNECTION for {item['name']}")

Expand Down
2 changes: 1 addition & 1 deletion aioradio/psycopg2.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import psycopg2


async def establish_psycopg2_connection(
def establish_psycopg2_connection(
host: str,
user: str,
password: str,
Expand Down
10 changes: 5 additions & 5 deletions aioradio/pyodbc.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
]


async def get_unixodbc_driver_path(paths: List[str]) -> Union[str, None]:
def get_unixodbc_driver_path(paths: List[str]) -> Union[str, None]:
"""Check the file system for the unixodbc driver.
Args:
Expand All @@ -40,7 +40,7 @@ async def get_unixodbc_driver_path(paths: List[str]) -> Union[str, None]:
return driver_path


async def establish_pyodbc_connection(
def establish_pyodbc_connection(
host: str,
user: str,
pwd: str,
Expand Down Expand Up @@ -80,7 +80,7 @@ async def establish_pyodbc_connection(
if OPERATING_SYSTEM == 'Windows':
verified_driver = driver
else:
verified_driver = driver if driver else await get_unixodbc_driver_path(UNIXODBC_DRIVER_PATHS)
verified_driver = driver if driver else get_unixodbc_driver_path(UNIXODBC_DRIVER_PATHS)
if verified_driver is None:
raise FileNotFoundError('Unable to locate unixodbc driver file: libtdsodbc.so')

Expand All @@ -101,7 +101,7 @@ async def establish_pyodbc_connection(
return pyodbc.connect(conn_string, autocommit=autocommit)


async def pyodbc_query_fetchone(conn: pyodbc.Connection, query: str) -> Union[List[Any], None]:
def pyodbc_query_fetchone(conn: pyodbc.Connection, query: str) -> Union[List[Any], None]:
"""Execute pyodbc query and fetchone, see
https://github.com/mkleehammer/pyodbc/wiki/Cursor.
Expand All @@ -120,7 +120,7 @@ async def pyodbc_query_fetchone(conn: pyodbc.Connection, query: str) -> Union[Li
return result


async def pyodbc_query_fetchall(conn: pyodbc.Connection, query: str) -> Union[List[Any], None]:
def pyodbc_query_fetchall(conn: pyodbc.Connection, query: str) -> Union[List[Any], None]:
"""Execute pyodbc query and fetchone, see
https://github.com/mkleehammer/pyodbc/wiki/Cursor.
Expand Down
22 changes: 11 additions & 11 deletions aioradio/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,25 +1,25 @@
aioboto3==9.2.2
aiobotocore==1.4.2
aioboto3==9.3.1
aiobotocore==2.1.0
aiojobs==0.3.0
boto3==1.17.106
ddtrace==0.57.0
boto3==1.20.24
ddtrace==0.58.1
fakeredis==1.7.0
flask==2.0.2
flask-cors==3.0.10
httpx==0.20.0
httpx==0.22.0
mandrill==1.0.60
moto==2.3.1
numpy==1.22.0
orjson==3.6.5
pre-commit==2.16.0
moto==3.0.3
numpy==1.22.2
orjson==3.6.6
pre-commit==2.17.0
psycopg2-binary==2.9.3
pylint==2.12.2
pyodbc==4.0.32
pysmb==1.2.7
pytest==6.2.5
pytest==7.0.0
pytest-asyncio==0.16.0
pytest-cov==3.0.0
python-json-logger==2.0.2
redis==3.5.3
twine==3.7.1
twine==3.8.0
wheel==0.37.1
2 changes: 1 addition & 1 deletion aioradio/tests/pyodbc_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
async def test_bad_unixodbc_driver():
"""Test using a bad unixodbc_driver that the proper exception is raised."""

await establish_pyodbc_connection(host='unknown', port=5123, user='psuedo', pwd='no-way-jose', driver='/usr/lib/bogus.so')
establish_pyodbc_connection(host='unknown', port=5123, user='psuedo', pwd='no-way-jose', driver='/usr/lib/bogus.so')


async def test_pyodbc_query_fetchone_and_fetchall():
Expand Down
23 changes: 11 additions & 12 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
long_description = fileobj.read()

setup(name='aioradio',
version='0.16.2',
version='0.17.0',
description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more',
long_description=long_description,
long_description_content_type="text/markdown",
Expand All @@ -19,30 +19,29 @@
'aioradio/aws',
],
install_requires=[
'aioboto3>=9.2.2',
'aiobotocore>=1.4.2',
'aioboto3>=9.3.1',
'aiobotocore>=2.1.0',
'aiojobs>=0.3.0',
'boto3==1.17.106',
'ddtrace>=0.57.0',
'boto3==1.20.24',
'ddtrace>=0.58.1',
'fakeredis>=1.7.0',
'httpx>=0.19.0',
'mandrill>=1.0.60',
'numpy>=1.19',
'orjson>=3.6.5',
'orjson>=3.6.6',
'psycopg2-binary==2.9.3',
'pysmb>=1.2.7',
'python-json-logger>=2.0.2',
'redis==3.5.3',
'xlrd==2.0.1'
'redis==3.5.3'
],
include_package_data=True,
tests_require=[
'flask>=2.0.2',
'flask-cors>=3.0.10',
'moto>=2.3.1',
'pre-commit>=2.16.0',
'pylint>=2.11.2',
'pytest>=6.2.5',
'moto>=3.0.3',
'pre-commit>=2.7.0',
'pylint>=2.10.2',
'pytest>=7.0.0',
'pytest-asyncio>=0.16.0',
'pytest-cov>=3.0.0'
],
Expand Down

0 comments on commit 331437f

Please sign in to comment.