diff --git a/README.md b/README.md index a186626..397382f 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Add `--upgrade` option to update existing package to a new version Specify package link in your `requirements.txt`: ```txt -git+https://github.com/gismart/bi-utils@0.16.1#egg=bi-utils-gismart +git+https://github.com/gismart/bi-utils@0.16.2#egg=bi-utils-gismart ``` ### Usage diff --git a/bi_utils/aws/db.py b/bi_utils/aws/db.py index 6fcadb0..7b5945a 100644 --- a/bi_utils/aws/db.py +++ b/bi_utils/aws/db.py @@ -91,16 +91,17 @@ def download_files( database: Optional[str] = None, host: Optional[str] = None, retries: int = 0, + max_chunk_size_mb: int = 6000, add_timestamp_dir: bool = True, add_s3_timestamp_dir: bool = True, ) -> Sequence[str]: """Copy data from RedShift to S3 and download csv or parquet files up to 6.2 GB""" - + max_chunk_size_opt = f"MAXFILESIZE {max_chunk_size_mb} MB" if file_format.lower() == "csv": - unload_options = ["CSV", "HEADER", "GZIP", "PARALLEL ON"] + unload_options = ["CSV", "HEADER", "GZIP", "PARALLEL ON", max_chunk_size_opt] elif file_format.lower() == "parquet": separator = None - unload_options = ["PARQUET", "PARALLEL ON"] + unload_options = ["PARQUET", "PARALLEL ON", max_chunk_size_opt] else: raise ValueError(f"{file_format} file format is not supported") if delete_s3_before: @@ -220,6 +221,7 @@ def download_data( database: Optional[str] = None, host: Optional[str] = None, retries: int = 0, + max_chunk_size_mb: int = 6000, remove_files: bool = True, delete_s3_before: bool = False, delete_s3_after: bool = True, @@ -238,6 +240,7 @@ def download_data( database=database, host=host, retries=retries, + max_chunk_size_mb=max_chunk_size_mb, delete_s3_before=delete_s3_before, delete_s3_after=delete_s3_after, add_timestamp_dir=add_timestamp_dir, diff --git a/setup.py b/setup.py index 55fdecb..c8e74cf 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setuptools.setup( name="bi-utils-gismart", - version="0.16.1", + version="0.16.2", author="gismart", author_email="info@gismart.com", description="Utils for BI team",