From 1cac50c9e3f7743b3f12bb9a3005cec864d92706 Mon Sep 17 00:00:00 2001 From: Zhengguo Yang Date: Wed, 19 Jun 2024 15:42:20 +0800 Subject: [PATCH] Fix potential double unload of JNI libraries on Java 11+ Previously, the addHookForLibUnloading method added a shutdown hook unconditionally to unload JNI libraries. This could potentially lead to double unload issues on Java 11 and later versions where the JVM automatically handles JNI library cleanup. This commit modifies addHookForLibUnloading to conditionally add the unload hook based on the JVM version: - For Java 11 and later, the method adds a no-op hook since the JVM handles the cleanup automatically. - For Java versions prior to 11, the method adds the provided hook to ensure proper cleanup. This change ensures compatibility and prevents potential double unload issues on Java 11 and later. --- .../spark/util/GlutenShutdownManager.scala | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/gluten-core/src/main/scala/org/apache/spark/util/GlutenShutdownManager.scala b/gluten-core/src/main/scala/org/apache/spark/util/GlutenShutdownManager.scala index 4bfa4ccec12b..bec6e8aaf8b7 100644 --- a/gluten-core/src/main/scala/org/apache/spark/util/GlutenShutdownManager.scala +++ b/gluten-core/src/main/scala/org/apache/spark/util/GlutenShutdownManager.scala @@ -16,6 +16,10 @@ */ package org.apache.spark.util +import org.apache.gluten.backendsapi.BackendsApiManager + +import org.apache.commons.lang3.{JavaVersion, SystemUtils} + object GlutenShutdownManager { def addHook(hook: () => Unit): AnyRef = { @@ -23,7 +27,17 @@ object GlutenShutdownManager { } def addHookForLibUnloading(hook: () => Unit): AnyRef = { - ShutdownHookManager.addShutdownHook(ShutdownHookManager.SPARK_CONTEXT_SHUTDOWN_PRIORITY)(hook) + ShutdownHookManager.addShutdownHook(ShutdownHookManager.SPARK_CONTEXT_SHUTDOWN_PRIORITY) { + if ( + BackendsApiManager.getBackendName == "velox" && SystemUtils.isJavaVersionAtLeast( + JavaVersion.JAVA_11) + ) { + // In Java 11 and later, the JVM automatically handles the cleanup of JNI libraries + () => Unit + } else { + hook + } + } } def addHookForTempDirRemoval(hook: () => Unit): AnyRef = {