[VL][Core] Turn off InputFileNameReplaceRule by default

apache · Jun 20, 2024 · 35695d1 · 35695d1
1 parent 79e1d58
commit 35695d1
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 8 deletions.
diff --git a/...ends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/...ends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -805,11 +805,14 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
    *
    * @return
    */
-  override def genExtendedColumnarValidationRules(): List[SparkSession => Rule[SparkPlan]] = List(
-    BloomFilterMightContainJointRewriteRule.apply,
-    ArrowScanReplaceRule.apply,
-    InputFileNameReplaceRule.apply
-  )
+  override def genExtendedColumnarValidationRules(): List[SparkSession => Rule[SparkPlan]] = {
+    val buf: ListBuffer[SparkSession => Rule[SparkPlan]] =
+      ListBuffer(BloomFilterMightContainJointRewriteRule.apply, ArrowScanReplaceRule.apply)
+    if (GlutenConfig.getConf.enableInputFileNameReplaceRule) {
+      buf += InputFileNameReplaceRule.apply
+    }
+    buf.result
+  }
 
   /**
    * Generate extended columnar pre-rules.

diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -624,9 +624,13 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest {
   }
 
   test("Test input_file_name function") {
-    runQueryAndCompare("""SELECT input_file_name(), l_orderkey
-                         | from lineitem limit 100""".stripMargin) {
-      checkGlutenOperatorMatch[ProjectExecTransformer]
+    withSQLConf(
+      "spark.gluten.sql.enableInputFileNameReplaceRule" -> "true"
+    ) {
+      runQueryAndCompare("""SELECT input_file_name(), l_orderkey
+                           | from lineitem limit 100""".stripMargin) {
+        checkGlutenOperatorMatch[ProjectExecTransformer]
+      }
     }
   }
 

diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -37,6 +37,7 @@ case class GlutenNumaBindingInfo(
 class GlutenConfig(conf: SQLConf) extends Logging {
   import GlutenConfig._
 
+  def enableInputFileNameReplaceRule: Boolean = conf.getConf(INPUT_FILE_NAME_REPLACE_RULE_ENABLED)
   def enableAnsiMode: Boolean = conf.ansiEnabled
 
   def enableGluten: Boolean = conf.getConf(GLUTEN_ENABLED)
@@ -750,6 +751,16 @@ object GlutenConfig {
       .booleanConf
       .createWithDefault(GLUTEN_ENABLE_BY_DEFAULT)
 
+  val INPUT_FILE_NAME_REPLACE_RULE_ENABLED =
+    buildConf("spark.gluten.sql.enableInputFileNameReplaceRule")
+      .internal()
+      .doc(
+        "Experimental: This config apply for velox backend to specify whether to enable " +
+          "inputFileNameReplaceRule to support offload input_file_name " +
+          "expression to native.")
+      .booleanConf
+      .createWithDefault(false)
+
   // FIXME the option currently controls both JVM and native validation against a Substrait plan.
   val NATIVE_VALIDATION_ENABLED =
     buildConf("spark.gluten.sql.enable.native.validation")