Merge pull request #43 from gismart/recipes

Add dict_merge func and default secret id const

README.md

@@ -15,7 +15,7 @@ Add `--upgrade` option to update existing package to a new version
 Specify package link in your `requirements.txt`:
 
 ```txt
-git+https://github.com/gismart/bi-utils@0.15.3#egg=bi-utils-gismart
+git+https://github.com/gismart/bi-utils@0.16.0#egg=bi-utils-gismart
 ```
 
 ### Usage

bi_utils/__init__.py

@@ -5,6 +5,7 @@
 from .queue_exporter import QueueExporter
 from .aws import db, s3, connection
 from .files import data_filename
+from .recipes import dict_merge
 from .decorators import retry
 from .sql import get_query
 from .qa import df_test

bi_utils/aws/connection.py

@@ -8,11 +8,12 @@
 from .locopy import locopy
 
 
+DEFAULT_SECRET_ID = "prod/redshift/bi_utils"
 logger = logging.getLogger(__name__)
 cached_creds: Dict[str, dict] = {}
 
 
-def get_creds(secret_id: str = "prod/redshift/analytics") -> dict:
+def get_creds(secret_id: str = DEFAULT_SECRET_ID) -> dict:
     """Get AWS credentials"""
     creds = cached_creds.get(secret_id)
     if not creds:
@@ -27,7 +28,7 @@ def get_creds(secret_id: str = "prod/redshift/analytics") -> dict:
 
 
 def get_redshift(
-    secret_id: str = "prod/redshift/analytics",
+    secret_id: str = DEFAULT_SECRET_ID,
     database: Optional[str] = None,
     host: Optional[str] = None,
 ) -> locopy.Redshift:
@@ -48,7 +49,7 @@ def get_redshift(
 
 
 def create_engine(
-    secret_id: str = "prod/redshift/analytics",
+    secret_id: str = DEFAULT_SECRET_ID,
     drivername: str = "postgresql+psycopg2",
     database: Optional[str] = None,
     host: Optional[str] = None,
@@ -72,7 +73,7 @@ def create_engine(
 
 def connect(
     schema: Optional[str] = None,
-    secret_id: str = "prod/redshift/analytics",
+    secret_id: str = DEFAULT_SECRET_ID,
     database: Optional[str] = None,
     host: Optional[str] = None,
 ) -> psycopg2.extensions.connection:

bi_utils/aws/db.py

@@ -26,7 +26,7 @@ def upload_file(
     bucket_dir: str = "dwh/temp",
     columns: Optional[Sequence] = None,
     delete_s3_after: bool = True,
-    secret_id: str = "prod/redshift/analytics",
+    secret_id: str = connection.DEFAULT_SECRET_ID,
     database: Optional[str] = None,
     host: Optional[str] = None,
     retries: int = 0,
@@ -87,7 +87,7 @@ def download_files(
     bucket_dir: str = "dwh/temp",
     delete_s3_before: bool = False,
     delete_s3_after: bool = True,
-    secret_id: str = "prod/redshift/analytics",
+    secret_id: str = connection.DEFAULT_SECRET_ID,
     database: Optional[str] = None,
     host: Optional[str] = None,
     retries: int = 0,
@@ -155,7 +155,7 @@ def upload_data(
     bucket_dir: str = "dwh/temp",
     columns: Optional[Sequence] = None,
     remove_file: bool = False,
-    secret_id: str = "prod/redshift/analytics",
+    secret_id: str = connection.DEFAULT_SECRET_ID,
     database: Optional[str] = None,
     host: Optional[str] = None,
     retries: int = 0,
@@ -211,7 +211,7 @@ def download_data(
     parse_bools: Optional[Sequence[str]] = None,
     dtype: Optional[dict] = None,
     chunking: bool = False,
-    secret_id: str = "prod/redshift/analytics",
+    secret_id: str = connection.DEFAULT_SECRET_ID,
     database: Optional[str] = None,
     host: Optional[str] = None,
     retries: int = 0,
@@ -308,7 +308,7 @@ def update(
     schema: str,
     params_set: dict,
     params_where: Optional[dict],
-    secret_id: str = "prod/redshift/analytics",
+    secret_id: str = connection.DEFAULT_SECRET_ID,
     database: Optional[str] = None,
     host: Optional[str] = None,
 ) -> None:
@@ -330,7 +330,7 @@ def update(
 def delete(
     table: str,
     schema: str,
-    secret_id: str = "prod/redshift/analytics",
+    secret_id: str = connection.DEFAULT_SECRET_ID,
     database: Optional[str] = None,
     host: Optional[str] = None,
     **conditions: Any,
@@ -353,7 +353,7 @@ def delete(
 def get_columns(
     table: str,
     schema: str,
-    secret_id: str = "prod/redshift/analytics",
+    secret_id: str = connection.DEFAULT_SECRET_ID,
     database: Optional[str] = None,
     host: Optional[str] = None,
 ) -> Sequence[str]:

bi_utils/queue_exporter.py

@@ -77,7 +77,7 @@ def export_df(
         delete_file_after: bool = False,
         delete_s3_after: bool = False,
         partition_cols: Optional[Sequence] = None,
-        secret_id: str = "prod/redshift/analytics",
+        secret_id: str = aws.connection.DEFAULT_SECRET_ID,
     ) -> None:
         """
         Save dataframe to `filepath` if `s3_bucket`, `s3_bucket_dir`, `schema`, `table` not passed
@@ -121,7 +121,7 @@ def export_file(
         table: Optional[str] = None,
         delete_file_after: bool = False,
         delete_s3_after: bool = False,
-        secret_id: str = "prod/redshift/analytics",
+        secret_id: str = aws.connection.DEFAULT_SECRET_ID,
     ) -> None:
         """
         Export file to S3 if `s3_bucket` and `s3_bucket_dir` passed
@@ -161,7 +161,7 @@ def _export_df(
         delete_file_after: bool = False,
         delete_s3_after: bool = False,
         partition_cols: Optional[Sequence] = None,
-        secret_id: str = "prod/redshift/analytics",
+        secret_id: str = aws.connection.DEFAULT_SECRET_ID,
     ) -> None:
         filename = os.path.basename(file_path)
         if columns:
@@ -211,7 +211,7 @@ def _export_file(
         table: Optional[str] = None,
         delete_file_after: bool = False,
         delete_s3_after: bool = False,
-        secret_id: str = "prod/redshift/analytics",
+        secret_id: str = aws.connection.DEFAULT_SECRET_ID,
     ) -> None:
         if schema and table and (".csv" in file_path.lower() or ".parquet" in file_path.lower()):
             aws.db.upload_file(

bi_utils/recipes.py

@@ -0,0 +1,9 @@
+def dict_merge(receiver: dict, updater: dict) -> dict:
+    """Update receiver dict with updater dict's values recursively"""
+    receiver = receiver.copy()
+    for k in updater:
+        if (k in receiver and isinstance(receiver[k], dict) and isinstance(updater[k], dict)):
+            receiver[k] = dict_merge(receiver[k], updater[k])
+        else:
+            receiver[k] = updater[k]
+    return receiver

setup.py

@@ -9,7 +9,7 @@
 
 setuptools.setup(
     name="bi-utils-gismart",
-    version="0.15.3",
+    version="0.16.0",
     author="gismart",
     author_email="[email protected]",
     description="Utils for BI team",

tests/test_recipes.py

@@ -0,0 +1,23 @@
+import pytest
+
+from bi_utils import recipes
+
+
+@pytest.mark.parametrize(
+    "reciever, updater, expected_dict",
+    [
+        (
+            {"a": {"one": 1, "two": [2, 2]}, "b": 3},
+            {"a": {"one": 11}},
+            {"a": {"one": 11, "two": [2, 2]}, "b": 3},
+        ),
+        (
+            {"a": {"one": 1, "two": [2, 2]}, "b": 3},
+            {"a": {"two": [2, 2, 2]}},
+            {"a": {"one": 1, "two": [2, 2, 2]}, "b": 3},
+        ),
+    ],
+)
+def test_dict_merge(reciever, updater, expected_dict):
+    result = recipes.dict_merge(reciever, updater)
+    assert result == expected_dict