Merge pull request #136 from nrccua/DS-502-update-mlflow-flask-flask-…

…cors-werkzeug-to-rem

Ds 502 update mlflow, flask, flask-cors & werkzeug to remove vunerability alert

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ default_language_version:
     python: python3.11
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
     -   id: check-added-large-files
     -   id: check-ast

HISTORY.rst

@@ -3,6 +3,28 @@ History
 =======
 
 
+v0.20.23 (2024-07-25)
+
+* Update install requires grpcio==1.62.2 & grpcio-status==1.62.2.
+
+
+v0.20.22 (2024-07-25)
+
+* Update install require numpy==1.26.4.
+
+
+v0.20.21 (2024-07-25)
+
+* Fix broken dependencies.
+
+
+v0.20.20 (2024-07-25)
+
+* Update mlflow, flask, flask-cors & werkzeug to remove vulnerability alert.
+* Update python libs.
+* Remove python lib dominodatalab.
+
+
 v0.20.19 (2024-05-06)
 
 * Add option to create polars LazyFrame from databricks query via pyarrow.

aioradio/ds_utils.py

@@ -529,33 +529,6 @@ def get_ftp_connection(secret_id, port=139, is_direct_tcp=False, env='sandbox'):
     return conn
 
 
-def monitor_domino_run(domino, run_id, sleep_time=10):
-    """Monitor domino job run and return True/False depending if job was
-    successful."""
-
-    status = None
-    while status is None:
-        sleep(sleep_time)
-        result = domino.runs_status(run_id)
-        if result['status'] in ["Finished", "Succeeded"]:
-            status = True
-            break
-        if result['status'] in ["Failed", "Error"]:
-            status = False
-            break
-
-    return status
-
-
-def get_domino_connection(secret_id, project, host, env='sandbox'):
-    """Get domino connection."""
-
-    from domino import Domino
-    secret_client = get_boto3_session(env).client("secretsmanager", region_name='us-east-1')
-    api_key = secret_client.get_secret_value(SecretId=secret_id)['SecretString']
-    return Domino(project=project, api_key=api_key, host=host)
-
-
 ######################## Postgres or MSSQL Connection Classes #######################
 
 

aioradio/requirements.txt

@@ -1,38 +1,42 @@
-aioboto3==12.4.0
-aiojobs==1.2.1
+aioboto3==13.1.1
+aiojobs==1.3.0
 backoff==2.2.1
-boto3==1.34.69
-botocore==1.34.69
+boto3==1.34.131
+botocore==1.34.131
 cython==3.0.10
 databricks-connect==14.3.1
 ddtrace==2.6.5
-dominodatalab==1.4.0
-fakeredis==2.22.0
+fakeredis==2.23.3
 faust-cchardet==2.1.19
-flask==2.1.2
-flask-cors==3.0.10
-haversine==2.8.0
+flask==3.0.3
+flask-cors==4.0.1
+grpcio==1.62.2
+grpcio-status==1.62.2
+haversine==2.8.1
 httpx==0.27.0
 mandrill==1.0.60
-mlflow==2.10.2
-moto==3.1.18
+mlflow==2.14.3
+moto==4.2.14
+numpy==1.26.4
 openpyxl==3.0.10
 orjson==3.9.15
 pandas==2.2.2
-polars==0.20.23
-pre-commit==3.7.0
+pkginfo==1.10.0
+polars==1.2.1
+pre-commit==3.7.1
+protobuf==4.25.4
 psycopg2-binary==2.9.9
 pyarrow==15.0.2
-pylint==3.1.0
+pylint==3.2.6
 pyodbc==5.1.0 --no-binary=pyodbc
 pysmb==1.2.9.1
 pyspark==3.4.3
 pytest==8.1.2
 pytest-asyncio==0.21.1
 pytest-cov==5.0.0
 python-json-logger==2.0.7
-redis==5.0.4
-twine==5.0.0
+redis==5.0.7
+twine==5.1.1
 typing_extensions==4.11.0
-werkzeug==2.1.2
+werkzeug==3.0.3
 wheel==0.43.0

conftest.py

@@ -199,8 +199,8 @@ async def sqs_queue_url(sqs_client):
 
     async def _f(region_name, queue_name):
         nonlocal _queue_url
-        fifo = "true" if queue_name.lower().endswith('.fifo') else "false"
-        response = await sqs_client.create_queue(QueueName=queue_name, Attributes={"FifoQueue": fifo})
+        attributes = {"FifoQueue": 'true', 'ContentBasedDeduplication': 'true'} if queue_name.lower().endswith('.fifo') else {}
+        response = await sqs_client.create_queue(QueueName=queue_name, Attributes=attributes)
         queue_url = response['QueueUrl']
         _queue_url = queue_url
         assert_status_code(response, 200)

setup.py

@@ -3,11 +3,11 @@
 
 from setuptools import setup
 
-with open('README.md', 'r') as fileobj:
+with open('README.md', 'r', encoding='utf8') as fileobj:
     long_description = fileobj.read()
 
 setup(name='aioradio',
-    version='0.20.19',
+    version='0.20.23',
     description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more',
     long_description=long_description,
     long_description_content_type="text/markdown",
@@ -21,40 +21,44 @@
     ],
     install_requires=[
         'cython>=0.29.33',
-        'aioboto3==12.4.0',
+        'aioboto3==13.1.1',
         'aiojobs>=1.0.0',
         'backoff>=2.1.2',
-        'botocore==1.34.69',
-        'boto3==1.34.69',
+        'botocore==1.34.131',
+        'boto3==1.34.131',
         'ddtrace>=0.60.1',
         'faust-cchardet>=2.1.18',
         'fakeredis>=2.20.0',
+        'grpcio==1.62.2',
+        'grpcio-status==1.62.2',
         'haversine>=2.8.0',
         'httpx>=0.23.0',
         'mandrill>=1.0.60',
         'mlflow>=2.10.2',
-        'numpy>=1.19',
+        'numpy==1.26.4',
         'openpyxl==3.0.10',
         'orjson>=3.6.8',
         'pandas>=1.3.5',
+        'pkginfo==1.10.0',
         'polars>=0.19.12',
+        'protobuf==4.25.4',
         'pyarrow>=13.0.0',
         'pysmb>=1.2.7',
         'python-json-logger>=2.0.2',
         'redis>=5.0.1'
     ],
     include_package_data=True,
     tests_require=[
-        'flask==2.1.2',
-        'flask-cors>=3.0.10',
-        'moto==3.1.18',
+        'flask==3.0.3',
+        'flask-cors>=4.0.1',
+        'moto==4.2.14',
         'pre-commit>=2.15.0',
         'pylint>=2.13.8',
         'pytest>=7.0.1',
         'pytest-asyncio>=0.15.1',
         'pytest-cov>=3.0.0',
         'typing_extensions>=4.10.0',
-        'werkzeug==2.1.2'
+        'werkzeug==3.0.3'
     ],
     zip_safe=False,
     classifiers=[