Merge pull request #125 from nrccua/DS-479-avoid-instantiating-spark-…

…if-databricks-connect-in Avoid instantiating spark if databricks-connect installed
nrccua · Mar 12, 2024 · 1a4eb3e · 1a4eb3e
2 parents 0951e40 + c2843ee
commit 1a4eb3e
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 2 deletions.
diff --git a/HISTORY.rst b/HISTORY.rst
@@ -3,6 +3,11 @@ History
 =======
 
 
+v0.20.12 (2024-03-12)
+
+* Avoid instantiating spark if databricks-connect installed.
+
+
 v0.20.11 (2024-03-12)
 
 * Add pyspark back in as install requirement.

diff --git a/aioradio/ds_utils.py b/aioradio/ds_utils.py
@@ -10,6 +10,8 @@
 # pylint: disable=too-many-boolean-expressions
 # pylint: disable=unnecessary-comprehension
 # pylint: disable=unused-argument
+# pylint: disable=unused-import
+# pylint: disable=used-before-assignment
 
 import base64
 import csv
@@ -46,7 +48,10 @@
 c_handler.setFormatter(c_format)
 logger.addHandler(c_handler)
 
-spark = SparkSession.builder.getOrCreate()
+try:
+    from databricks.connect import DatabricksSession
+except ImportError:
+    spark = SparkSession.builder.getOrCreate()
 
 
 ############################### Databricks functions ################################

diff --git a/setup.py b/setup.py
@@ -7,7 +7,7 @@
     long_description = fileobj.read()
 
 setup(name='aioradio',
-    version='0.20.11',
+    version='0.20.12',
     description='Generic asynchronous i/o python utilities for AWS services (SQS, S3, DynamoDB, Secrets Manager), Redis, MSSQL (pyodbc), JIRA and more',
     long_description=long_description,
     long_description_content_type="text/markdown",