From 6fa72ee131512c682818aa48e267cf168c51a442 Mon Sep 17 00:00:00 2001 From: Jia Ke Date: Wed, 14 Aug 2024 23:19:05 +0800 Subject: [PATCH] Init ExpressionUtil.extendedExpressionTransformer in executor side --- .../scala/org/apache/gluten/GlutenPlugin.scala | 18 +++++++++--------- .../gluten/expression/ExpressionMappings.scala | 5 ++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index dbf9279091877..a25367f6a5067 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -74,15 +74,10 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { BackendsApiManager.initialize() BackendsApiManager.getListenerApiInstance.onDriverStart(sc, pluginContext) GlutenListenerFactory.addToSparkListenerBus(sc) - - val expressionExtensionTransformer = ExpressionUtil.extendedExpressionTransformer( - conf.get(GlutenConfig.GLUTEN_EXTENDED_EXPRESSION_TRAN_CONF, "") - ) - - if (expressionExtensionTransformer != null) { - ExpressionMappings.expressionExtensionTransformer = expressionExtensionTransformer - } - + ExpressionMappings.expressionExtensionTransformer = + ExpressionUtil.extendedExpressionTransformer( + conf.get(GlutenConfig.GLUTEN_EXTENDED_EXPRESSION_TRAN_CONF, "") + ) Collections.emptyMap() } @@ -277,6 +272,11 @@ private[gluten] class GlutenExecutorPlugin extends ExecutorPlugin { // TODO categorize the APIs by driver's or executor's BackendsApiManager.initialize() BackendsApiManager.getListenerApiInstance.onExecutorStart(ctx) + + ExpressionMappings.expressionExtensionTransformer = + ExpressionUtil.extendedExpressionTransformer( + conf.get(GlutenConfig.GLUTEN_EXTENDED_EXPRESSION_TRAN_CONF, "") + ) } /** Clean up and terminate this plugin. For example: close the native engine. */ diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index f2bb4a90621af..e0628f11102d2 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -19,7 +19,7 @@ package org.apache.gluten.expression import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.expression.ExpressionNames._ -import org.apache.gluten.extension.{DefaultExpressionExtensionTransformer, ExpressionExtensionTrait} +import org.apache.gluten.extension.ExpressionExtensionTrait import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.sql.catalyst.expressions._ @@ -354,6 +354,5 @@ object ExpressionMappings { .toMap[Class[_], String] } - var expressionExtensionTransformer: ExpressionExtensionTrait = - DefaultExpressionExtensionTransformer() + var expressionExtensionTransformer: ExpressionExtensionTrait = _ }