From 6e27602d5261c993f448605c21579756c22c7228 Mon Sep 17 00:00:00 2001 From: Tim Reichard Date: Thu, 14 Dec 2023 08:33:25 -0600 Subject: [PATCH] Add stage_table optional argument to merge_spark_df_in_db ds.utils func --- HISTORY.rst | 8 ++++++++ aioradio/ds_utils.py | 8 ++++---- aioradio/requirements.txt | 12 ++++++------ aioradio/tests/dynamodb_test.py | 15 ++++++++++----- aioradio/tests/sqs_test.py | 9 +++++++-- conftest.py | 1 - setup.py | 8 ++++---- 7 files changed, 39 insertions(+), 22 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b9de550..426c33e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,14 @@ History ======= +v0.20.6 (2023-12-14) + +* Add stage_table optional argument to merge_spark_df_in_db ds.utils function. +* Update aioboto3==12.1.0. +* Update boto3==1.33.1. +* Update botocore==1.33.1. + + v0.20.5 (2023-12-08) * Add function ese_db_catalog in ds_utils. diff --git a/aioradio/ds_utils.py b/aioradio/ds_utils.py index b54ada4..c494b88 100644 --- a/aioradio/ds_utils.py +++ b/aioradio/ds_utils.py @@ -95,11 +95,11 @@ def does_db_table_exists(name): return exists -def merge_spark_df_in_db(df, target, on, partition_by=None): +def merge_spark_df_in_db(df, target, on, partition_by=None, stage_table=None): """Convert spark DF to staging table than merge with target table in Databricks.""" - stage = f"{target}_stage" + stage = f"{target}_stage" if stage_table is None else stage_table if not does_db_table_exists(target): if partition_by is None: @@ -123,11 +123,11 @@ def merge_spark_df_in_db(df, target, on, partition_by=None): raise -def merge_pandas_df_in_db(df, target, on, partition_by=None): +def merge_pandas_df_in_db(df, target, on, partition_by=None, stage_table=None): """Convert pandas DF to staging table than merge with target table in Databricks.""" - stage = f"{target}_stage" + stage = f"{target}_stage" if stage_table is None else stage_table for col, dtype in df.dtypes.apply(lambda x: x.name).to_dict().items(): if dtype == 'object': diff --git a/aioradio/requirements.txt b/aioradio/requirements.txt index 2b39e58..d9e832c 100644 --- a/aioradio/requirements.txt +++ b/aioradio/requirements.txt @@ -1,8 +1,8 @@ -aioboto3==11.3.1 +aioboto3==12.1.0 aiojobs==1.2.1 backoff==2.2.1 -boto3==1.28.17 -botocore==1.31.17 +boto3==1.33.1 +botocore==1.33.1 cython==3.0.6 ddtrace==1.11.2 dominodatalab==1.2.4 @@ -18,15 +18,15 @@ openpyxl==3.0.10 orjson==3.8.10 pandas==2.1.4 polars==0.19.19 -pre-commit==3.5.0 +pre-commit==3.6.0 psycopg2-binary==2.9.9 pyarrow==14.0.1 -pylint==3.0.2 +pylint==3.0.3 pyodbc==4.0.39 --no-binary=pyodbc pysmb==1.2.9.1 pyspark==3.4.1 pytest==7.4.3 -pytest-asyncio==0.23.2 +pytest-asyncio==0.21.1 pytest-cov==4.1.0 python-json-logger==2.0.7 redis==5.0.1 diff --git a/aioradio/tests/dynamodb_test.py b/aioradio/tests/dynamodb_test.py index ac6045f..ab764e8 100644 --- a/aioradio/tests/dynamodb_test.py +++ b/aioradio/tests/dynamodb_test.py @@ -7,11 +7,16 @@ import pytest from boto3.dynamodb.conditions import Attr, Key -from aioradio.aws.dynamodb import (add_regions, batch_get_items_from_dynamo, - batch_write_to_dynamo, - get_list_of_dynamo_tables, - put_item_in_dynamo, query_dynamo, - scan_dynamo, update_item_in_dynamo) +from aioradio.aws.dynamodb import ( + add_regions, + batch_get_items_from_dynamo, + batch_write_to_dynamo, + get_list_of_dynamo_tables, + put_item_in_dynamo, + query_dynamo, + scan_dynamo, + update_item_in_dynamo +) # **************************************** # DO NOT CHANGE THE DB_TABLE OR REGION diff --git a/aioradio/tests/sqs_test.py b/aioradio/tests/sqs_test.py index 3a82cd5..bb7ed47 100644 --- a/aioradio/tests/sqs_test.py +++ b/aioradio/tests/sqs_test.py @@ -8,8 +8,13 @@ import orjson import pytest -from aioradio.aws.sqs import (add_regions, delete_messages, get_messages, - purge_messages, send_messages) +from aioradio.aws.sqs import ( + add_regions, + delete_messages, + get_messages, + purge_messages, + send_messages +) QUEUE = 'pytest' REGION = 'us-east-2' diff --git a/conftest.py b/conftest.py index ad6bc75..1b2e213 100644 --- a/conftest.py +++ b/conftest.py @@ -6,7 +6,6 @@ import aioboto3 import aiobotocore -import boto3 import pytest import pytest_asyncio from aiobotocore.config import AioConfig diff --git a/setup.py b/setup.py index e81db9d..115d229 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ long_description = fileobj.read() setup(name='aioradio', - version='0.20.5', + version='0.20.6', description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more', long_description=long_description, long_description_content_type="text/markdown", @@ -21,11 +21,11 @@ ], install_requires=[ 'cython>=0.29.33', - 'aioboto3==11.3.1', + 'aioboto3==12.1.0', 'aiojobs>=1.0.0', 'backoff>=2.1.2', - 'botocore==1.31.17', - 'boto3==1.28.17', + 'botocore==1.33.1', + 'boto3==1.33.1', 'ddtrace>=0.60.1', 'faust-cchardet>=2.1.18', 'fakeredis>=2.20.0',