Skip to content

Commit

Permalink
[GLUTEN-8462][CH] Fixed the loading of Components and Backend
Browse files Browse the repository at this point in the history
  • Loading branch information
gleonSun committed Jan 8, 2025
1 parent 799149e commit 57a1d1a
Show file tree
Hide file tree
Showing 4 changed files with 34 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,24 @@ public class ResourceUtil {
*/
public static List<String> getResources(final Pattern pattern) {
final List<String> buffer = new ArrayList<>();
final String classPath = System.getProperty("java.class.path");
String classPath = System.getProperty("java.class.path");
processClassPathElements(classPath, pattern, buffer);
if (buffer.isEmpty()) {
classPath = ResourceUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath();
processClassPathElements(classPath, pattern, buffer);
}
return Collections.unmodifiableList(buffer);
}

private static void processClassPathElements(
String classPath, Pattern pattern, List<String> buffer) {
if (classPath == null || classPath.isEmpty()) {
return;
}
final String[] classPathElements = classPath.split(File.pathSeparator);
for (final String element : classPathElements) {
getResources(element, pattern, buffer);
}
return Collections.unmodifiableList(buffer);
}

private static void getResources(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging {
val glutenBuildInfo = new mutable.LinkedHashMap[String, String]()

val components = Component.sorted()
assert(components.nonEmpty)
glutenBuildInfo.put("Components", components.map(_.buildInfo().name).mkString(", "))
components.foreach {
comp =>
Expand Down Expand Up @@ -269,7 +270,9 @@ private[gluten] class GlutenExecutorPlugin extends ExecutorPlugin {
/** Initialize the executor plugin. */
override def init(ctx: PluginContext, extraConf: util.Map[String, String]): Unit = {
// Initialize Backend.
Component.sorted().foreach(_.onExecutorStart(ctx))
val components = Component.sorted()
assert(components.nonEmpty)
components.foreach(_.onExecutorStart(ctx))
}

/** Clean up and terminate this plugin. For example: close the native engine. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.apache.gluten.component

import org.apache.gluten.backend.Backend
import org.apache.gluten.exception.GlutenException
import org.apache.gluten.extension.injector.Injector

import org.scalatest.BeforeAndAfterAll
Expand All @@ -36,6 +37,8 @@ class ComponentSuite extends AnyFunSuite with BeforeAndAfterAll {
private val e = new DummyComponentE()
e.ensureRegistered()

private val classPath: String = System.getProperty("java.class.path")

test("Load order - sanity") {
val possibleOrders =
Set(
Expand All @@ -44,7 +47,7 @@ class ComponentSuite extends AnyFunSuite with BeforeAndAfterAll {
Seq(b, a, c, d, e),
Seq(b, a, d, c, e)
)

System.setProperty("java.class.path", "")
assert(possibleOrders.contains(Component.sorted()))
}

Expand All @@ -53,6 +56,18 @@ class ComponentSuite extends AnyFunSuite with BeforeAndAfterAll {
new DummyBackendA().ensureRegistered()
}
}

test("Discover without component class") {
System.setProperty("java.class.path", classPath)
assertThrows[GlutenException](Discovery.discoverAll())
}

// Gluten attempts to load from the "classes" directory instead of "test-classes,"
// but the META-INF is not located in the "classes" directory.
test("Discover without component class again") {
System.setProperty("java.class.path", "")
assert(Discovery.discoverAll().isEmpty)
}
}

object ComponentSuite {
Expand Down

0 comments on commit 57a1d1a

Please sign in to comment.