Merge pull request #127 from nrccua/DS-481-fix-bug-with-spark-not-bei…

…ng-instantiated Fix bug with spark not being instantiated
nrccua · Mar 15, 2024 · d74001d · d74001d
2 parents dededb2 + a23d084
commit d74001d
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 1 deletion.
diff --git a/HISTORY.rst b/HISTORY.rst
@@ -3,6 +3,11 @@ History
 =======
 
 
+v0.20.14 (2024-03-15)
+
+* Fix bug with spark not being instantiated.
+
+
 v0.20.13 (2024-03-15)
 
 * Use spark df.toPandas() function instead of private function converted spark df -> pandas -> polars.

diff --git a/aioradio/ds_utils.py b/aioradio/ds_utils.py
@@ -50,6 +50,7 @@
 
 try:
     from databricks.connect import DatabricksSession
+    spark = DatabricksSession.builder.getOrCreate()
 except ImportError:
     spark = SparkSession.builder.getOrCreate()
 

diff --git a/setup.py b/setup.py
@@ -7,7 +7,7 @@
     long_description = fileobj.read()
 
 setup(name='aioradio',
-    version='0.20.13',
+    version='0.20.14',
     description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more',
     long_description=long_description,
     long_description_content_type="text/markdown",