Merge pull request #119 from nrccua/DS-466-add-stage_table-argument-t…

…o-merge_spark_df_in_db

Add stage_table optional argument to merge_spark_df_in_db ds.utils func

HISTORY.rst

@@ -3,6 +3,14 @@ History
 =======
 
 
+v0.20.6 (2023-12-14)
+
+* Add stage_table optional argument to merge_spark_df_in_db ds.utils function.
+* Update aioboto3==12.1.0.
+* Update boto3==1.33.1.
+* Update botocore==1.33.1.
+
+
 v0.20.5 (2023-12-08)
 
 * Add function ese_db_catalog in ds_utils.

aioradio/ds_utils.py

@@ -95,11 +95,11 @@ def does_db_table_exists(name):
     return exists
 
 
-def merge_spark_df_in_db(df, target, on, partition_by=None):
+def merge_spark_df_in_db(df, target, on, partition_by=None, stage_table=None):
     """Convert spark DF to staging table than merge with target table in
     Databricks."""
 
-    stage = f"{target}_stage"
+    stage = f"{target}_stage" if stage_table is None else stage_table
 
     if not does_db_table_exists(target):
         if partition_by is None:
@@ -123,11 +123,11 @@ def merge_spark_df_in_db(df, target, on, partition_by=None):
             raise
 
 
-def merge_pandas_df_in_db(df, target, on, partition_by=None):
+def merge_pandas_df_in_db(df, target, on, partition_by=None, stage_table=None):
     """Convert pandas DF to staging table than merge with target table in
     Databricks."""
 
-    stage = f"{target}_stage"
+    stage = f"{target}_stage" if stage_table is None else stage_table
 
     for col, dtype in df.dtypes.apply(lambda x: x.name).to_dict().items():
         if dtype == 'object':

aioradio/requirements.txt

@@ -1,8 +1,8 @@
-aioboto3==11.3.1
+aioboto3==12.1.0
 aiojobs==1.2.1
 backoff==2.2.1
-boto3==1.28.17
-botocore==1.31.17
+boto3==1.33.1
+botocore==1.33.1
 cython==3.0.6
 ddtrace==1.11.2
 dominodatalab==1.2.4
@@ -18,15 +18,15 @@ openpyxl==3.0.10
 orjson==3.8.10
 pandas==2.1.4
 polars==0.19.19
-pre-commit==3.5.0
+pre-commit==3.6.0
 psycopg2-binary==2.9.9
 pyarrow==14.0.1
-pylint==3.0.2
+pylint==3.0.3
 pyodbc==4.0.39 --no-binary=pyodbc
 pysmb==1.2.9.1
 pyspark==3.4.1
 pytest==7.4.3
-pytest-asyncio==0.23.2
+pytest-asyncio==0.21.1
 pytest-cov==4.1.0
 python-json-logger==2.0.7
 redis==5.0.1

aioradio/tests/dynamodb_test.py

@@ -7,11 +7,16 @@
 import pytest
 from boto3.dynamodb.conditions import Attr, Key
 
-from aioradio.aws.dynamodb import (add_regions, batch_get_items_from_dynamo,
-                                   batch_write_to_dynamo,
-                                   get_list_of_dynamo_tables,
-                                   put_item_in_dynamo, query_dynamo,
-                                   scan_dynamo, update_item_in_dynamo)
+from aioradio.aws.dynamodb import (
+    add_regions,
+    batch_get_items_from_dynamo,
+    batch_write_to_dynamo,
+    get_list_of_dynamo_tables,
+    put_item_in_dynamo,
+    query_dynamo,
+    scan_dynamo,
+    update_item_in_dynamo
+)
 
 # ****************************************
 # DO NOT CHANGE THE DB_TABLE OR REGION

aioradio/tests/sqs_test.py

@@ -8,8 +8,13 @@
 import orjson
 import pytest
 
-from aioradio.aws.sqs import (add_regions, delete_messages, get_messages,
-                              purge_messages, send_messages)
+from aioradio.aws.sqs import (
+    add_regions,
+    delete_messages,
+    get_messages,
+    purge_messages,
+    send_messages
+)
 
 QUEUE = 'pytest'
 REGION = 'us-east-2'

conftest.py

@@ -6,7 +6,6 @@
 
 import aioboto3
 import aiobotocore
-import boto3
 import pytest
 import pytest_asyncio
 from aiobotocore.config import AioConfig

setup.py

@@ -7,7 +7,7 @@
     long_description = fileobj.read()
 
 setup(name='aioradio',
-    version='0.20.5',
+    version='0.20.6',
     description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more',
     long_description=long_description,
     long_description_content_type="text/markdown",
@@ -21,11 +21,11 @@
     ],
     install_requires=[
         'cython>=0.29.33',
-        'aioboto3==11.3.1',
+        'aioboto3==12.1.0',
         'aiojobs>=1.0.0',
         'backoff>=2.1.2',
-        'botocore==1.31.17',
-        'boto3==1.28.17',
+        'botocore==1.33.1',
+        'boto3==1.33.1',
         'ddtrace>=0.60.1',
         'faust-cchardet>=2.1.18',
         'fakeredis>=2.20.0',