From 3c22a2ecf839946c34a92a6370e965db54b793da Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 5 Sep 2024 15:18:41 +0800 Subject: [PATCH] [VL] Minor follow-ups for #6942 --- backends-velox/pom.xml | 2 +- .../backendsapi/velox/VeloxListenerApi.scala | 27 +++++++------- .../gluten/test/VeloxBackendTestBase.java | 4 +-- .../velox/VeloxListenerApiSuite.scala | 35 +++++++++++++++++++ package/pom.xml | 14 ++++---- 5 files changed, 58 insertions(+), 24 deletions(-) create mode 100644 backends-velox/src/test/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApiSuite.scala diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml index c678168252b6..88fa460452b2 100755 --- a/backends-velox/pom.xml +++ b/backends-velox/pom.xml @@ -187,7 +187,7 @@ ${cpp.releases.dir} - org/apache/gluten/${platform}/${arch} + ${platform}/${arch} ${resource.dir} diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index d2c4293906e4..7b33e1279039 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -138,8 +138,8 @@ class VeloxListenerApi extends ListenerApi with Logging { JniLibLoader.loadFromPath(libPath, false) } else { val baseLibName = conf.get(GlutenConfig.GLUTEN_LIB_NAME, "gluten") - loader.load(s"$baseLibPackage/${System.mapLibraryName(baseLibName)}", false) - loader.load(s"$baseLibPackage/${System.mapLibraryName(VeloxBackend.BACKEND_NAME)}", false) + loader.load(s"$platformLibDir/${System.mapLibraryName(baseLibName)}", false) + loader.load(s"$platformLibDir/${System.mapLibraryName(VeloxBackend.BACKEND_NAME)}", false) } // Initial native backend with configurations. @@ -152,18 +152,6 @@ class VeloxListenerApi extends ListenerApi with Logging { GlutenRowSplitter.setInstance(new VeloxRowSplitter()) } - private lazy val baseLibPackage: String = { - val osName = System.getProperty("os.name") match { - case n if n.contains("Linux") => "linux" - case n if n.contains("Mac") => "darwin" - case _ => - // Default to linux - "linux" - } - val arch = System.getProperty("os.arch") - s"org/apache/gluten/$osName/$arch" - } - private def shutdown(): Unit = { // TODO shutdown implementation in velox to release resources } @@ -174,6 +162,17 @@ object VeloxListenerApi { // As spark conf may change when active Spark session is recreated. private val driverInitialized: AtomicBoolean = new AtomicBoolean(false) private val executorInitialized: AtomicBoolean = new AtomicBoolean(false) + private val platformLibDir: String = { + val osName = System.getProperty("os.name") match { + case n if n.contains("Linux") => "linux" + case n if n.contains("Mac") => "darwin" + case _ => + // Default to linux + "linux" + } + val arch = System.getProperty("os.arch") + s"$osName/$arch" + } private def inLocalMode(conf: SparkConf): Boolean = { SparkResourceUtil.isLocalMaster(conf) diff --git a/backends-velox/src/test/java/org/apache/gluten/test/VeloxBackendTestBase.java b/backends-velox/src/test/java/org/apache/gluten/test/VeloxBackendTestBase.java index 1cbaefba8fb8..a026a2ee8f34 100644 --- a/backends-velox/src/test/java/org/apache/gluten/test/VeloxBackendTestBase.java +++ b/backends-velox/src/test/java/org/apache/gluten/test/VeloxBackendTestBase.java @@ -38,12 +38,12 @@ public static void setup() { api.onDriverStart(mockSparkContext(), mockPluginContext()); } - private static SparkContext mockSparkContext() { + public static SparkContext mockSparkContext() { // Not yet implemented. return null; } - private static PluginContext mockPluginContext() { + public static PluginContext mockPluginContext() { return new PluginContext() { @Override public MetricRegistry metricRegistry() { diff --git a/backends-velox/src/test/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApiSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApiSuite.scala new file mode 100644 index 000000000000..e3156fea750f --- /dev/null +++ b/backends-velox/src/test/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApiSuite.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.backendsapi.velox + +import org.apache.gluten.test.VeloxBackendTestBase + +import org.scalatest.funsuite.AnyFunSuite + +class VeloxListenerApiSuite extends AnyFunSuite { + test("Unsupported arch") { + System.setProperty("os.arch", "unknown-arch") + val api = new VeloxListenerApi() + val error = intercept[RuntimeException] { + api.onDriverStart( + VeloxBackendTestBase.mockSparkContext(), + VeloxBackendTestBase.mockPluginContext()) + } + assert(error.getMessage.contains("FileNotFoundException")) + assert(error.getMessage.contains("unknown-arch/libgluten.so")) + } +} diff --git a/package/pom.xml b/package/pom.xml index ac0754c38e7e..fc72fe93de84 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -98,7 +98,7 @@ org.apache.gluten - gluten-delta + gluten-delta ${project.version} @@ -196,12 +196,12 @@ META-INF/*.SF META-INF/*.DSA - META-INF/*.RSA - META-INF/DEPENDENCIES - META-INF/LICENSE.txt - META-INF/NOTICE.txt - LICENSE.txt - NOTICE.txt + META-INF/*.RSA + META-INF/DEPENDENCIES + META-INF/LICENSE.txt + META-INF/NOTICE.txt + LICENSE.txt + NOTICE.txt