chore: Update pyproject.toml dependencies for pyspark and spark-expec…

…tations

pyproject.toml

@@ -278,10 +278,16 @@ version = ["pyspark35"]
 
 [tool.hatch.envs.hatch-test.overrides]
 matrix.version.extra-dependencies = [
-  { value = "pyspark>=3.3,<3.4; spark-expectations>=2.1.0", if = [
+  { value = "pyspark>=3.3,<3.4", if = [
     "pyspark33",
   ] },
-  { value = "pyspark>=3.4,<3.5; spark-expectations>=2.1.0", if = [
+  { value = "spark-expectations>=2.1.0", if = [
+    "pyspark33",
+  ] },
+  { value = "pyspark>=3.4,<3.5", if = [
+    "pyspark34",
+  ] },
+  { value = "spark-expectations>=2.1.0", if = [
     "pyspark34",
   ] },
   { value = "pyspark>=3.5,<3.6", if = [

src/koheesio/integrations/spark/dq/spark_expectations.py

@@ -4,19 +4,21 @@
 
 from typing import Any, Dict, Optional, Union
 
+import pyspark
+from pydantic import Field
+from pyspark.sql import DataFrame
 from spark_expectations.config.user_config import Constants as user_config
 from spark_expectations.core.expectations import (
     SparkExpectations,
     WrappedDataFrameWriter,
 )
 
-from pydantic import Field
-
-from pyspark.sql import DataFrame
-
 from koheesio.spark.transformations import Transformation
 from koheesio.spark.writers import BatchOutputMode
 
+if pyspark.__version__.startswith("3.5"):
+    raise ImportError("Spark Expectations is not supported for Spark 3.5")
+
 
 class SparkExpectationsTransformation(Transformation):
     """