Skip to content

Commit

Permalink
[VL] Minor follow-ups for apache#6942
Browse files Browse the repository at this point in the history
  • Loading branch information
zhztheplayer committed Sep 5, 2024
1 parent d289b54 commit 3c22a2e
Show file tree
Hide file tree
Showing 5 changed files with 58 additions and 24 deletions.
2 changes: 1 addition & 1 deletion backends-velox/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@
<resources>
<resource>
<directory>${cpp.releases.dir}</directory>
<targetPath>org/apache/gluten/${platform}/${arch}</targetPath>
<targetPath>${platform}/${arch}</targetPath>
</resource>
<resource>
<directory>${resource.dir}</directory>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,8 @@ class VeloxListenerApi extends ListenerApi with Logging {
JniLibLoader.loadFromPath(libPath, false)
} else {
val baseLibName = conf.get(GlutenConfig.GLUTEN_LIB_NAME, "gluten")
loader.load(s"$baseLibPackage/${System.mapLibraryName(baseLibName)}", false)
loader.load(s"$baseLibPackage/${System.mapLibraryName(VeloxBackend.BACKEND_NAME)}", false)
loader.load(s"$platformLibDir/${System.mapLibraryName(baseLibName)}", false)
loader.load(s"$platformLibDir/${System.mapLibraryName(VeloxBackend.BACKEND_NAME)}", false)
}

// Initial native backend with configurations.
Expand All @@ -152,18 +152,6 @@ class VeloxListenerApi extends ListenerApi with Logging {
GlutenRowSplitter.setInstance(new VeloxRowSplitter())
}

private lazy val baseLibPackage: String = {
val osName = System.getProperty("os.name") match {
case n if n.contains("Linux") => "linux"
case n if n.contains("Mac") => "darwin"
case _ =>
// Default to linux
"linux"
}
val arch = System.getProperty("os.arch")
s"org/apache/gluten/$osName/$arch"
}

private def shutdown(): Unit = {
// TODO shutdown implementation in velox to release resources
}
Expand All @@ -174,6 +162,17 @@ object VeloxListenerApi {
// As spark conf may change when active Spark session is recreated.
private val driverInitialized: AtomicBoolean = new AtomicBoolean(false)
private val executorInitialized: AtomicBoolean = new AtomicBoolean(false)
private val platformLibDir: String = {
val osName = System.getProperty("os.name") match {
case n if n.contains("Linux") => "linux"
case n if n.contains("Mac") => "darwin"
case _ =>
// Default to linux
"linux"
}
val arch = System.getProperty("os.arch")
s"$osName/$arch"
}

private def inLocalMode(conf: SparkConf): Boolean = {
SparkResourceUtil.isLocalMaster(conf)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,12 @@ public static void setup() {
api.onDriverStart(mockSparkContext(), mockPluginContext());
}

private static SparkContext mockSparkContext() {
public static SparkContext mockSparkContext() {
// Not yet implemented.
return null;
}

private static PluginContext mockPluginContext() {
public static PluginContext mockPluginContext() {
return new PluginContext() {
@Override
public MetricRegistry metricRegistry() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.gluten.backendsapi.velox

import org.apache.gluten.test.VeloxBackendTestBase

import org.scalatest.funsuite.AnyFunSuite

class VeloxListenerApiSuite extends AnyFunSuite {
test("Unsupported arch") {
System.setProperty("os.arch", "unknown-arch")
val api = new VeloxListenerApi()
val error = intercept[RuntimeException] {
api.onDriverStart(
VeloxBackendTestBase.mockSparkContext(),
VeloxBackendTestBase.mockPluginContext())
}
assert(error.getMessage.contains("FileNotFoundException"))
assert(error.getMessage.contains("unknown-arch/libgluten.so"))
}
}
14 changes: 7 additions & 7 deletions package/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@
<dependencies>
<dependency>
<groupId>org.apache.gluten</groupId>
<artifactId>gluten-delta</artifactId>
<artifactId>gluten-delta</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
Expand Down Expand Up @@ -196,12 +196,12 @@
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/DEPENDENCIES</exclude>
<exclude>META-INF/LICENSE.txt</exclude>
<exclude>META-INF/NOTICE.txt</exclude>
<exclude>LICENSE.txt</exclude>
<exclude>NOTICE.txt</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/DEPENDENCIES</exclude>
<exclude>META-INF/LICENSE.txt</exclude>
<exclude>META-INF/NOTICE.txt</exclude>
<exclude>LICENSE.txt</exclude>
<exclude>NOTICE.txt</exclude>
</excludes>
</filter>
</filters>
Expand Down

0 comments on commit 3c22a2e

Please sign in to comment.