[GLUTEN-7031][CORE] Move JNI / exception utilities to gluten-core (ap…

…ache#7134)
sharkdtu · Oct 14, 2024 · f9e3aeb · f9e3aeb
1 parent 2712395
commit f9e3aeb
Show file tree

Hide file tree

Showing 23 changed files with 27 additions and 255 deletions.
diff --git a/...ds-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala b/...ds-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
@@ -22,7 +22,8 @@ import org.apache.gluten.execution.CHBroadcastBuildSideCache
 import org.apache.gluten.execution.datasource.{GlutenOrcWriterInjects, GlutenParquetWriterInjects, GlutenRowSplitter}
 import org.apache.gluten.expression.UDFMappings
 import org.apache.gluten.extension.ExpressionExtensionTrait
-import org.apache.gluten.vectorized.{CHNativeExpressionEvaluator, JniLibLoader}
+import org.apache.gluten.jni.JniLibLoader
+import org.apache.gluten.vectorized.CHNativeExpressionEvaluator
 
 import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.api.plugin.PluginContext

diff --git a/...khouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala b/...khouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala
@@ -17,8 +17,8 @@
 package org.apache.spark.sql.execution.benchmarks
 
 import org.apache.gluten.GlutenConfig
+import org.apache.gluten.jni.JniLibLoader
 import org.apache.gluten.utils.UTSystemParameters
-import org.apache.gluten.vectorized.JniLibLoader
 
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.delta.DeltaLog

diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -21,9 +21,9 @@ import org.apache.gluten.backendsapi.ListenerApi
 import org.apache.gluten.execution.datasource.{GlutenOrcWriterInjects, GlutenParquetWriterInjects, GlutenRowSplitter}
 import org.apache.gluten.expression.UDFMappings
 import org.apache.gluten.init.NativeBackendInitializer
+import org.apache.gluten.jni.{JniLibLoader, JniWorkspace}
 import org.apache.gluten.udf.UdfJniWrapper
 import org.apache.gluten.utils._
-import org.apache.gluten.vectorized.{JniLibLoader, JniWorkspace}
 
 import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.api.plugin.PluginContext

diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala
@@ -18,7 +18,7 @@ package org.apache.gluten.utils
 
 import org.apache.gluten.GlutenConfig
 import org.apache.gluten.exception.GlutenException
-import org.apache.gluten.vectorized.JniLibLoader
+import org.apache.gluten.jni.JniLibLoader
 
 import org.apache.spark.SparkConf
 

diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
@@ -16,7 +16,7 @@
  */
 package org.apache.gluten.utils
 
-import org.apache.gluten.vectorized.JniLibLoader
+import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderCentos7 extends SharedLibraryLoader {
   override def loadLib(loader: JniLibLoader): Unit = {

diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
@@ -16,7 +16,7 @@
  */
 package org.apache.gluten.utils
 
-import org.apache.gluten.vectorized.JniLibLoader
+import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderCentos8 extends SharedLibraryLoader {
   override def loadLib(loader: JniLibLoader): Unit = {

diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
@@ -16,7 +16,7 @@
  */
 package org.apache.gluten.utils
 
-import org.apache.gluten.vectorized.JniLibLoader
+import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderCentos9 extends SharedLibraryLoader {
   override def loadLib(loader: JniLibLoader): Unit = {

diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
@@ -16,7 +16,7 @@
  */
 package org.apache.gluten.utils
 
-import org.apache.gluten.vectorized.JniLibLoader
+import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderDebian11 extends SharedLibraryLoader {
   override def loadLib(loader: JniLibLoader): Unit = {

diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
@@ -16,7 +16,7 @@
  */
 package org.apache.gluten.utils
 
-import org.apache.gluten.vectorized.JniLibLoader
+import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderDebian12 extends SharedLibraryLoader {
   override def loadLib(loader: JniLibLoader): Unit = {

diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
@@ -16,7 +16,7 @@
  */
 package org.apache.gluten.utils
 
-import org.apache.gluten.vectorized.JniLibLoader
+import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderUbuntu2004 extends SharedLibraryLoader {
   override def loadLib(loader: JniLibLoader): Unit = {

diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
@@ -16,7 +16,7 @@
  */
 package org.apache.gluten.utils
 
-import org.apache.gluten.vectorized.JniLibLoader
+import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader {
   override def loadLib(loader: JniLibLoader): Unit = {

diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.expression
 import org.apache.gluten.backendsapi.velox.VeloxBackendSettings
 import org.apache.gluten.exception.{GlutenException, GlutenNotSupportException}
 import org.apache.gluten.expression._
-import org.apache.gluten.vectorized.JniWorkspace
+import org.apache.gluten.jni.JniWorkspace
 
 import org.apache.spark.{SparkConf, SparkFiles}
 import org.apache.spark.deploy.SparkHadoopUtil

diff --git a/...che/gluten/exception/GlutenException.java → ...che/gluten/exception/GlutenException.java b/...che/gluten/exception/GlutenException.java → ...che/gluten/exception/GlutenException.java
diff --git a/...pache/gluten/vectorized/JniLibLoader.java → ...a/org/apache/gluten/jni/JniLibLoader.java b/...pache/gluten/vectorized/JniLibLoader.java → ...a/org/apache/gluten/jni/JniLibLoader.java
@@ -14,11 +14,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.vectorized;
+package org.apache.gluten.jni;
 
 import org.apache.gluten.exception.GlutenException;
 
-import org.apache.spark.util.GlutenShutdownManager;
+import org.apache.spark.util.SparkShutdownManagerUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -51,7 +51,7 @@ public class JniLibLoader {
       Collections.synchronizedSet(new LinkedHashSet<>());
 
   static {
-    GlutenShutdownManager.addHookForLibUnloading(
+    SparkShutdownManagerUtil.addHookForLibUnloading(
         () -> {
           forceUnloadAll();
           return BoxedUnit.UNIT;

diff --git a/...pache/gluten/vectorized/JniWorkspace.java → ...a/org/apache/gluten/jni/JniWorkspace.java b/...pache/gluten/vectorized/JniWorkspace.java → ...a/org/apache/gluten/jni/JniWorkspace.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.vectorized;
+package org.apache.gluten.jni;
 
 import org.apache.gluten.exception.GlutenException;
 
@@ -46,7 +46,6 @@ public class JniWorkspace {
 
   private final String workDir;
   private final JniLibLoader jniLibLoader;
-  private final JniResourceHelper jniResourceHelper;
 
   private JniWorkspace(String rootDir) {
     try {
@@ -55,7 +54,6 @@ private JniWorkspace(String rootDir) {
       Path created = Files.createTempDirectory(root, "gluten-");
       this.workDir = created.toAbsolutePath().toString();
       this.jniLibLoader = new JniLibLoader(workDir);
-      this.jniResourceHelper = new JniResourceHelper(workDir);
       LOG.info("JNI workspace {} created in root directory {}", workDir, rootDir);
     } catch (Exception e) {
       throw new GlutenException(e);
@@ -101,22 +99,6 @@ public static void enableDebug(String debugDir) {
     }
   }
 
-  // For testing
-  private static boolean isDebugEnabled() {
-    synchronized (DEFAULT_INSTANCE_INIT_LOCK) {
-      return DEFAULT_INSTANCE != null
-          && DEBUG_INSTANCE != null
-          && DEFAULT_INSTANCE == DEBUG_INSTANCE;
-    }
-  }
-
-  // For testing
-  private static void resetDefaultInstance() {
-    synchronized (DEFAULT_INSTANCE_INIT_LOCK) {
-      DEFAULT_INSTANCE = null;
-    }
-  }
-
   public static JniWorkspace getDefault() {
     synchronized (DEFAULT_INSTANCE_INIT_LOCK) {
       if (DEFAULT_INSTANCE == null) {
@@ -138,8 +120,4 @@ public String getWorkDir() {
   public JniLibLoader libLoader() {
     return jniLibLoader;
   }
-
-  public JniResourceHelper resourceHelper() {
-    return jniResourceHelper;
-  }
 }
diff --git a/...pache/spark/util/SparkDirectoryUtil.scala → ...pache/spark/util/SparkDirectoryUtil.scala b/...pache/spark/util/SparkDirectoryUtil.scala → ...pache/spark/util/SparkDirectoryUtil.scala
@@ -16,12 +16,11 @@
  */
 package org.apache.spark.util
 
-import org.apache.spark.SparkConf
-import org.apache.spark.internal.Logging
-
 import _root_.org.apache.gluten.exception.GlutenException
 import org.apache.commons.io.FileUtils
 import org.apache.commons.lang3.StringUtils
+import org.apache.spark.SparkConf
+import org.apache.spark.internal.Logging
 
 import java.io.{File, IOException}
 import java.nio.file.Paths
@@ -35,7 +34,7 @@ class SparkDirectoryUtil private (val roots: Array[String]) extends Logging {
     rootDir =>
       try {
         val localDir = Utils.createDirectory(rootDir, "gluten")
-        GlutenShutdownManager.addHookForTempDirRemoval(
+        SparkShutdownManagerUtil.addHookForTempDirRemoval(
           () => {
             try FileUtils.forceDelete(localDir)
             catch {

diff --git a/...he/spark/util/GlutenShutdownManager.scala → ...spark/util/SparkShutdownManagerUtil.scala b/...he/spark/util/GlutenShutdownManager.scala → ...spark/util/SparkShutdownManagerUtil.scala
@@ -16,8 +16,7 @@
  */
 package org.apache.spark.util
 
-object GlutenShutdownManager {
-
+object SparkShutdownManagerUtil {
   def addHook(hook: () => Unit): AnyRef = {
     ShutdownHookManager.addShutdownHook(ShutdownHookManager.DEFAULT_SHUTDOWN_PRIORITY)(hook)
   }

diff --git a/gluten-data/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java b/gluten-data/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java
@@ -19,7 +19,7 @@
 import org.apache.gluten.GlutenConfig;
 import org.apache.gluten.backendsapi.BackendsApiManager;
 
-import org.apache.spark.util.GlutenShutdownManager;
+import org.apache.spark.util.SparkShutdownManagerUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -46,7 +46,7 @@ public static void initializeBackend(scala.collection.Map<String, String> conf)
       return;
     }
     initialize0(conf);
-    GlutenShutdownManager.addHook(
+    SparkShutdownManagerUtil.addHook(
         () -> {
           shutdown();
           return BoxedUnit.UNIT;