diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala index 60dc3dad0b87..3e1443f3a59a 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala @@ -22,7 +22,8 @@ import org.apache.gluten.execution.CHBroadcastBuildSideCache import org.apache.gluten.execution.datasource.{GlutenOrcWriterInjects, GlutenParquetWriterInjects, GlutenRowSplitter} import org.apache.gluten.expression.UDFMappings import org.apache.gluten.extension.ExpressionExtensionTrait -import org.apache.gluten.vectorized.{CHNativeExpressionEvaluator, JniLibLoader} +import org.apache.gluten.jni.JniLibLoader +import org.apache.gluten.vectorized.CHNativeExpressionEvaluator import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.api.plugin.PluginContext diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala index ab541901ef98..0467ae2814fa 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala +++ b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala @@ -17,8 +17,8 @@ package org.apache.spark.sql.execution.benchmarks import org.apache.gluten.GlutenConfig +import org.apache.gluten.jni.JniLibLoader import org.apache.gluten.utils.UTSystemParameters -import org.apache.gluten.vectorized.JniLibLoader import org.apache.spark.SparkConf import org.apache.spark.sql.delta.DeltaLog diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index d2c4293906e4..b7b9ac263a50 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -21,9 +21,9 @@ import org.apache.gluten.backendsapi.ListenerApi import org.apache.gluten.execution.datasource.{GlutenOrcWriterInjects, GlutenParquetWriterInjects, GlutenRowSplitter} import org.apache.gluten.expression.UDFMappings import org.apache.gluten.init.NativeBackendInitializer +import org.apache.gluten.jni.{JniLibLoader, JniWorkspace} import org.apache.gluten.udf.UdfJniWrapper import org.apache.gluten.utils._ -import org.apache.gluten.vectorized.{JniLibLoader, JniWorkspace} import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.api.plugin.PluginContext diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala index 1f3ca30de9f5..28a40f4b3d8a 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala @@ -18,7 +18,7 @@ package org.apache.gluten.utils import org.apache.gluten.GlutenConfig import org.apache.gluten.exception.GlutenException -import org.apache.gluten.vectorized.JniLibLoader +import org.apache.gluten.jni.JniLibLoader import org.apache.spark.SparkConf diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala index 384d00fe6d55..b7b63acc769f 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.utils -import org.apache.gluten.vectorized.JniLibLoader +import org.apache.gluten.jni.JniLibLoader class SharedLibraryLoaderCentos7 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala index 0167ed4e2f51..d0b91c09bbf7 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.utils -import org.apache.gluten.vectorized.JniLibLoader +import org.apache.gluten.jni.JniLibLoader class SharedLibraryLoaderCentos8 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala index 92fd1f5c600c..7084c9e47fb0 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.utils -import org.apache.gluten.vectorized.JniLibLoader +import org.apache.gluten.jni.JniLibLoader class SharedLibraryLoaderCentos9 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala index 3aebb725bf39..5713c8522158 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.utils -import org.apache.gluten.vectorized.JniLibLoader +import org.apache.gluten.jni.JniLibLoader class SharedLibraryLoaderDebian11 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala index 80b20129f364..62eb813c5f3a 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.utils -import org.apache.gluten.vectorized.JniLibLoader +import org.apache.gluten.jni.JniLibLoader class SharedLibraryLoaderDebian12 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala index 150eae6a1cdf..bb9d7274784f 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.utils -import org.apache.gluten.vectorized.JniLibLoader +import org.apache.gluten.jni.JniLibLoader class SharedLibraryLoaderUbuntu2004 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala index 7183f13ddb02..e988ed6916f5 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.utils -import org.apache.gluten.vectorized.JniLibLoader +import org.apache.gluten.jni.JniLibLoader class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala index db166a51327c..17f661289ebc 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/expression/UDFResolver.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.expression import org.apache.gluten.backendsapi.velox.VeloxBackendSettings import org.apache.gluten.exception.{GlutenException, GlutenNotSupportException} import org.apache.gluten.expression._ -import org.apache.gluten.vectorized.JniWorkspace +import org.apache.gluten.jni.JniWorkspace import org.apache.spark.{SparkConf, SparkFiles} import org.apache.spark.deploy.SparkHadoopUtil diff --git a/gluten-substrait/src/main/java/org/apache/gluten/exception/GlutenException.java b/gluten-core/src/main/java/org/apache/gluten/exception/GlutenException.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/exception/GlutenException.java rename to gluten-core/src/main/java/org/apache/gluten/exception/GlutenException.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java b/gluten-core/src/main/java/org/apache/gluten/jni/JniLibLoader.java similarity index 98% rename from gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java rename to gluten-core/src/main/java/org/apache/gluten/jni/JniLibLoader.java index 7ca246a8454b..384fb7752840 100644 --- a/gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java +++ b/gluten-core/src/main/java/org/apache/gluten/jni/JniLibLoader.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.vectorized; +package org.apache.gluten.jni; import org.apache.gluten.exception.GlutenException; -import org.apache.spark.util.GlutenShutdownManager; +import org.apache.spark.util.SparkShutdownManagerUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,7 +51,7 @@ public class JniLibLoader { Collections.synchronizedSet(new LinkedHashSet<>()); static { - GlutenShutdownManager.addHookForLibUnloading( + SparkShutdownManagerUtil.addHookForLibUnloading( () -> { forceUnloadAll(); return BoxedUnit.UNIT; diff --git a/gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniWorkspace.java b/gluten-core/src/main/java/org/apache/gluten/jni/JniWorkspace.java similarity index 86% rename from gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniWorkspace.java rename to gluten-core/src/main/java/org/apache/gluten/jni/JniWorkspace.java index 810c945d35ab..a37aac03c583 100644 --- a/gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniWorkspace.java +++ b/gluten-core/src/main/java/org/apache/gluten/jni/JniWorkspace.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.vectorized; +package org.apache.gluten.jni; import org.apache.gluten.exception.GlutenException; @@ -46,7 +46,6 @@ public class JniWorkspace { private final String workDir; private final JniLibLoader jniLibLoader; - private final JniResourceHelper jniResourceHelper; private JniWorkspace(String rootDir) { try { @@ -55,7 +54,6 @@ private JniWorkspace(String rootDir) { Path created = Files.createTempDirectory(root, "gluten-"); this.workDir = created.toAbsolutePath().toString(); this.jniLibLoader = new JniLibLoader(workDir); - this.jniResourceHelper = new JniResourceHelper(workDir); LOG.info("JNI workspace {} created in root directory {}", workDir, rootDir); } catch (Exception e) { throw new GlutenException(e); @@ -101,22 +99,6 @@ public static void enableDebug(String debugDir) { } } - // For testing - private static boolean isDebugEnabled() { - synchronized (DEFAULT_INSTANCE_INIT_LOCK) { - return DEFAULT_INSTANCE != null - && DEBUG_INSTANCE != null - && DEFAULT_INSTANCE == DEBUG_INSTANCE; - } - } - - // For testing - private static void resetDefaultInstance() { - synchronized (DEFAULT_INSTANCE_INIT_LOCK) { - DEFAULT_INSTANCE = null; - } - } - public static JniWorkspace getDefault() { synchronized (DEFAULT_INSTANCE_INIT_LOCK) { if (DEFAULT_INSTANCE == null) { @@ -138,8 +120,4 @@ public String getWorkDir() { public JniLibLoader libLoader() { return jniLibLoader; } - - public JniResourceHelper resourceHelper() { - return jniResourceHelper; - } } diff --git a/gluten-substrait/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala b/gluten-core/src/main/java/org/apache/spark/util/SparkDirectoryUtil.scala similarity index 98% rename from gluten-substrait/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala rename to gluten-core/src/main/java/org/apache/spark/util/SparkDirectoryUtil.scala index 833575178c66..06d5eb8ae827 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala +++ b/gluten-core/src/main/java/org/apache/spark/util/SparkDirectoryUtil.scala @@ -16,12 +16,11 @@ */ package org.apache.spark.util -import org.apache.spark.SparkConf -import org.apache.spark.internal.Logging - import _root_.org.apache.gluten.exception.GlutenException import org.apache.commons.io.FileUtils import org.apache.commons.lang3.StringUtils +import org.apache.spark.SparkConf +import org.apache.spark.internal.Logging import java.io.{File, IOException} import java.nio.file.Paths @@ -35,7 +34,7 @@ class SparkDirectoryUtil private (val roots: Array[String]) extends Logging { rootDir => try { val localDir = Utils.createDirectory(rootDir, "gluten") - GlutenShutdownManager.addHookForTempDirRemoval( + SparkShutdownManagerUtil.addHookForTempDirRemoval( () => { try FileUtils.forceDelete(localDir) catch { diff --git a/gluten-substrait/src/main/scala/org/apache/spark/util/GlutenShutdownManager.scala b/gluten-core/src/main/java/org/apache/spark/util/SparkShutdownManagerUtil.scala similarity index 97% rename from gluten-substrait/src/main/scala/org/apache/spark/util/GlutenShutdownManager.scala rename to gluten-core/src/main/java/org/apache/spark/util/SparkShutdownManagerUtil.scala index 4bfa4ccec12b..291d591a0ef6 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/util/GlutenShutdownManager.scala +++ b/gluten-core/src/main/java/org/apache/spark/util/SparkShutdownManagerUtil.scala @@ -16,8 +16,7 @@ */ package org.apache.spark.util -object GlutenShutdownManager { - +object SparkShutdownManagerUtil { def addHook(hook: () => Unit): AnyRef = { ShutdownHookManager.addShutdownHook(ShutdownHookManager.DEFAULT_SHUTDOWN_PRIORITY)(hook) } diff --git a/gluten-data/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java b/gluten-data/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java index 4863f481f47d..1aafb0ca0494 100644 --- a/gluten-data/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java +++ b/gluten-data/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java @@ -19,7 +19,7 @@ import org.apache.gluten.GlutenConfig; import org.apache.gluten.backendsapi.BackendsApiManager; -import org.apache.spark.util.GlutenShutdownManager; +import org.apache.spark.util.SparkShutdownManagerUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +46,7 @@ public static void initializeBackend(scala.collection.Map conf) return; } initialize0(conf); - GlutenShutdownManager.addHook( + SparkShutdownManagerUtil.addHook( () -> { shutdown(); return BoxedUnit.UNIT; diff --git a/gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniResourceHelper.java b/gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniResourceHelper.java deleted file mode 100644 index 58917ae9ae2f..000000000000 --- a/gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniResourceHelper.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.vectorized; - -import org.apache.gluten.exception.GlutenException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.JarURLConnection; -import java.net.URL; -import java.net.URLConnection; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.Enumeration; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; - -public class JniResourceHelper { - private static final Logger LOG = LoggerFactory.getLogger(JniResourceHelper.class); - - private final String workDir; - private final Set jarExtracted = new HashSet<>(); - - private boolean headersExtracted = false; - - JniResourceHelper(String workDir) { - this.workDir = workDir; - } - - private static void extractHeaders0(String workDir) throws IOException { - if (workDir == null) { - workDir = System.getProperty("java.io.tmpdir"); - } - final String folderToLoad = "include"; - final URL connResource = JniResourceHelper.class.getClassLoader().getResource("include"); - if (connResource != null) { - final URLConnection urlConnection = connResource.openConnection(); - if (urlConnection instanceof JarURLConnection) { - final JarFile jarFile = ((JarURLConnection) urlConnection).getJarFile(); - extractResourcesToDirectory(jarFile, folderToLoad, workDir + "/" + "nativesql_include"); - } else { - // For Maven test only - String path = urlConnection.getURL().toString(); - if (urlConnection.getURL().toString().startsWith("file:")) { - // remove the prefix of "file:" from includePath - path = urlConnection.getURL().toString().substring(5); - } - final File folder = new File(path); - copyResourcesToDirectory(urlConnection, workDir + "/" + "nativesql_include", folder); - } - } else { - LOG.info("There is no include dir in jars."); - } - } - - private static void copyResourcesToDirectory( - URLConnection urlConnection, String destPath, File folder) { - for (final File fileEntry : Objects.requireNonNull(folder.listFiles())) { - String destFilePath = destPath + "/" + fileEntry.getName(); - File destFile = new File(destFilePath); - if (fileEntry.isDirectory()) { - destFile.mkdirs(); - copyResourcesToDirectory(urlConnection, destFilePath, fileEntry); - } else { - try { - Files.copy(fileEntry.toPath(), destFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - } catch (IOException e) { - LOG.error("Error copying file", e); - } - } - } - } - - private static void extractJar(String sourceJar, String workDir) throws IOException { - if (workDir == null) { - workDir = System.getProperty("java.io.tmpdir"); - } - final String folderToLoad = ""; - URL url = new URL("jar:file:" + sourceJar + "!/"); - final URLConnection urlConnection = url.openConnection(); - File workDirHandler = new File(workDir + "/tmp"); - if (!workDirHandler.exists()) { - workDirHandler.mkdirs(); - } - - if (urlConnection instanceof JarURLConnection) { - final JarFile jarFile = ((JarURLConnection) urlConnection).getJarFile(); - extractResourcesToDirectory(jarFile, folderToLoad, workDir + "/tmp/"); - } else { - throw new IOException(urlConnection.toString() + " is not JarUrlConnection"); - } - /* - * System.out.println("Current content under " + workDir + "/tmp/"); - * Files.list(new File(workDir + "/tmp/").toPath()).forEach(path -> { - * System.out.println(path); }); - */ - } - - private static void extractResourcesToDirectory(JarFile origJar, String jarPath, String destPath) - throws IOException { - for (Enumeration entries = origJar.entries(); entries.hasMoreElements(); ) { - JarEntry oneEntry = entries.nextElement(); - if (((Objects.equals(jarPath, "") && !oneEntry.getName().contains("META-INF")) - || (oneEntry.getName().startsWith(jarPath + "/"))) - && !oneEntry.isDirectory()) { - int rmLength = jarPath.isEmpty() ? 0 : jarPath.length() + 1; - if (Files.exists(Paths.get(destPath + "/" + oneEntry.getName().substring(rmLength)))) { - continue; - } - File destFile = new File(destPath + "/" + oneEntry.getName().substring(rmLength)); - File parentFile = destFile.getParentFile(); - if (parentFile != null) { - parentFile.mkdirs(); - } - - FileOutputStream outFile = new FileOutputStream(destFile); - InputStream inFile = origJar.getInputStream(oneEntry); - - try { - byte[] buffer = new byte[4 * 1024]; - - int s = 0; - while ((s = inFile.read(buffer)) > 0) { - outFile.write(buffer, 0, s); - } - } catch (IOException e) { - throw new IOException("Could not extract resource from jar", e); - } finally { - try { - inFile.close(); - } catch (IOException e) { - LOG.error("Error closing file", e); - } - try { - outFile.close(); - } catch (IOException e) { - LOG.error("Error closing file", e); - } - } - } - } - } - - public synchronized void extractHeaders() { - try { - if (headersExtracted) { - LOG.debug("Headers already extracted to work directory {}, skipping", workDir); - return; - } - LOG.info("Trying to extract headers to work directory {}", workDir); - extractHeaders0(workDir); - LOG.info("Successfully extracted headers to work directory {}", workDir); - headersExtracted = true; - } catch (Exception e) { - throw new GlutenException(e); - } - } - - public synchronized void extractJars(List jars) { - jars.stream() - .filter( - jar -> { - if (jarExtracted.contains(jar)) { - LOG.debug("Jar {} already extracted to work directory {}, skipping", jar, workDir); - return false; - } - return true; - }) - .forEach( - jar -> { - try { - LOG.info("Trying to extract jar {} to work directory {}", jar, workDir); - extractJar(jar, workDir); - LOG.info("Successfully extracted jar {} to work directory {}", jar, workDir); - jarExtracted.add(jar); - } catch (IOException e) { - throw new GlutenException(e); - } - }); - } -} diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index ef96994582c4..3db750dc20a4 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -19,8 +19,8 @@ package org.apache.spark.sql.execution.benchmarks import org.apache.gluten.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} import org.apache.gluten.extension.columnar.transition.Transitions +import org.apache.gluten.jni.JniLibLoader import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} -import org.apache.gluten.vectorized.JniLibLoader import org.apache.spark.SparkConf import org.apache.spark.benchmark.Benchmark diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index f61f7ebba9b2..706c30d67db8 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -19,8 +19,8 @@ package org.apache.spark.sql.execution.benchmarks import org.apache.gluten.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} import org.apache.gluten.extension.columnar.transition.Transitions +import org.apache.gluten.jni.JniLibLoader import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} -import org.apache.gluten.vectorized.JniLibLoader import org.apache.spark.SparkConf import org.apache.spark.benchmark.Benchmark diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index 471bdf1796cd..640b8cdc6f8f 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -19,8 +19,8 @@ package org.apache.spark.sql.execution.benchmarks import org.apache.gluten.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} import org.apache.gluten.extension.columnar.transition.Transitions +import org.apache.gluten.jni.JniLibLoader import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} -import org.apache.gluten.vectorized.JniLibLoader import org.apache.spark.SparkConf import org.apache.spark.benchmark.Benchmark diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index f61f7ebba9b2..706c30d67db8 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -19,8 +19,8 @@ package org.apache.spark.sql.execution.benchmarks import org.apache.gluten.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} import org.apache.gluten.extension.columnar.transition.Transitions +import org.apache.gluten.jni.JniLibLoader import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} -import org.apache.gluten.vectorized.JniLibLoader import org.apache.spark.SparkConf import org.apache.spark.benchmark.Benchmark