Update csv sniffer to read 8192 bytes instea of 4096

nrccua · Feb 13, 2023 · f439d6b · f439d6b
1 parent 8d28a39
commit f439d6b
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 3 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -15,7 +15,7 @@ repos:
     -   id: requirements-txt-fixer
     -   id: trailing-whitespace
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.11.4
+    rev: 5.12.0
     hooks:
     -   id: isort
 -   repo: https://github.com/myint/docformatter

diff --git a/HISTORY.rst b/HISTORY.rst
@@ -3,6 +3,11 @@ History
 =======
 
 
+v0.17.32 (2023-02-03)
+
+* Update csv sniffer to read 8192 bytes instea of 4096.
+
+
 v0.17.31 (2023-02-03)
 
 * Remove results from callback when error in LongRunningJobs.

diff --git a/aioradio/file_ingestion.py b/aioradio/file_ingestion.py
@@ -1744,7 +1744,7 @@ def tsv_to_records(path: str, encoding: str, delimiter: str, header: str) -> tup
     records = []
     with open(path, newline='', encoding=encoding) as csvfile:
 
-        dialect = csv.Sniffer().sniff(csvfile.read(4096))
+        dialect = csv.Sniffer().sniff(csvfile.read(8192))
         csvfile.seek(0)
 
         # remove any null characters in the file

diff --git a/setup.py b/setup.py
@@ -7,7 +7,7 @@
     long_description = fileobj.read()
 
 setup(name='aioradio',
-    version='0.17.31',
+    version='0.17.32',
     description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more',
     long_description=long_description,
     long_description_content_type="text/markdown",