From 79cdb11a818980bc54aebe28504335e87e48aff2 Mon Sep 17 00:00:00 2001 From: Bobby Wang Date: Thu, 23 Jan 2025 16:53:07 +0800 Subject: [PATCH] [SPARK-50959][ML][PYTHON] Swallow the exception of JavaWrapper.__del__ ### What changes were proposed in this pull request? ``` python from pyspark.ml.classification import LogisticRegression from pyspark.ml.linalg import Vectors from pyspark.sql import SparkSession spark = SparkSession.builder.master("local[*]").getOrCreate() dataset = spark.createDataFrame( [(Vectors.dense([0.0]), 0.0), (Vectors.dense([0.4]), 1.0), (Vectors.dense([0.5]), 0.0), (Vectors.dense([0.6]), 1.0), (Vectors.dense([1.0]), 1.0)] * 10, ["features", "label"]) lr = LogisticRegression() model = lr.fit(dataset) ``` The above code will raise exception at the end. Even I remove the `try_remote_del` from https://github.com/apache/spark/blob/master/python/pyspark/ml/wrapper.py#L58 , the issue is still there. ``` console Exception ignored in: _ Traceback (most recent call last): File "/home/xxx/work.d/spark/spark-master/python/pyspark/ml/util.py", line 254, in wrapped File "/home/xxx/work.d/spark/spark-master/python/pyspark/ml/wrapper.py", line 60, in __del__ ImportError: sys.meta_path is None, Python is likely shutting down Exception ignored in: _ Traceback (most recent call last): File "/home/xxx/work.d/spark/spark-master/python/pyspark/ml/util.py", line 254, in wrapped File "/home/xxx/work.d/spark/spark-master/python/pyspark/ml/wrapper.py", line 60, in __del__ ImportError: sys.meta_path is None, Python is likely shutting down ``` Looks like the `__del__` function of JavaWrapper is called after python shutting down and the python related modules is not functional. ### Why are the changes needed? Fix the bug, good user experience. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? CI passes ### Was this patch authored or co-authored using generative AI tooling? No Closes #49615 from wbo4958/del.bug. Authored-by: Bobby Wang Signed-off-by: Ruifeng Zheng (cherry picked from commit 74cf4d82edd78a1f0bdcb1b570cb0c4d581d12d2) Signed-off-by: Ruifeng Zheng --- python/pyspark/ml/wrapper.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/python/pyspark/ml/wrapper.py b/python/pyspark/ml/wrapper.py index 8fb38fc7ce180..40670fdb84a21 100644 --- a/python/pyspark/ml/wrapper.py +++ b/python/pyspark/ml/wrapper.py @@ -57,12 +57,15 @@ def __init__(self, java_obj: Optional["JavaObject"] = None): @try_remote_del def __del__(self) -> None: - from pyspark.core.context import SparkContext - - if SparkContext._active_spark_context and self._java_obj is not None: - SparkContext._active_spark_context._gateway.detach( # type: ignore[union-attr] - self._java_obj - ) + try: + from pyspark.core.context import SparkContext + + if SparkContext._active_spark_context and self._java_obj is not None: + SparkContext._active_spark_context._gateway.detach( # type: ignore[union-attr] + self._java_obj + ) + except Exception: + pass @classmethod def _create_from_java_class(cls: Type[JW], java_class: str, *args: Any) -> JW: