[VL][1.2] Port #7121 #7448 (#7988)

* [GLUTEN-6748][CORE] Search stack trace to infer adaptive execution context (#7121) Closes #6748 * [Core] fix GH security issues (#7448) Uncontrolled Resource Consumption vulnerability in Apache Commons IO. The org.apache.commons.io.input.XmlStreamReader class may excessively consume CPU resources when processing maliciously crafted input. This issue affects Apache Commons IO: from 2.0 before 2.14.0. Users are recommended to upgrade to version 2.14.0 or later, which fixes the issue. Signed-off-by: Yuan Zhou <[email protected]> --------- Signed-off-by: Yuan Zhou <[email protected]> Co-authored-by: PHILO-HE <[email protected]> Co-authored-by: Yuan <[email protected]>
apache · Nov 20, 2024 · ae56ca5 · ae56ca5
1 parent be17f7e
commit ae56ca5
Show file tree

Hide file tree

Showing 7 changed files with 11 additions and 32 deletions.
diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml
@@ -177,7 +177,7 @@
     <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>
-      <version>2.11.0</version>
+      <version>2.14.0</version>
       <scope>provided</scope>
     </dependency>
     <!-- Fasterxml -->

diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml
@@ -132,7 +132,7 @@
     <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>
-      <version>2.11.0</version>
+      <version>2.14.0</version>
       <scope>provided</scope>
     </dependency>
     <dependency>

diff --git a/gluten-core/pom.xml b/gluten-core/pom.xml
@@ -195,7 +195,7 @@
     <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>
-      <version>2.11.0</version>
+      <version>2.14.0</version>
       <scope>provided</scope>
     </dependency>
 

diff --git a/...re/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala b/...re/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala
@@ -44,14 +44,7 @@ class EnumeratedApplier(session: SparkSession)
   extends ColumnarRuleApplier
   with Logging
   with LogLevelUtil {
-  // An empirical value.
-  private val aqeStackTraceIndex =
-    if (scala.util.Properties.releaseVersion.exists(_.startsWith("2.12"))) {
-      16
-    } else {
-      14
-    }
-  private val adaptiveContext = AdaptiveContext(session, aqeStackTraceIndex)
+  private val adaptiveContext = AdaptiveContext(session)
 
   override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan =
     PhysicalPlanSelector.maybe(session, plan) {

diff --git a/...core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala b/...core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala
@@ -39,14 +39,7 @@ class HeuristicApplier(session: SparkSession)
   extends ColumnarRuleApplier
   with Logging
   with LogLevelUtil {
-  // This is an empirical value, may need to be changed for supporting other versions of spark.
-  private val aqeStackTraceIndex =
-    if (scala.util.Properties.releaseVersion.exists(_.startsWith("2.12"))) {
-      19
-    } else {
-      17
-    }
-  private val adaptiveContext = AdaptiveContext(session, aqeStackTraceIndex)
+  private val adaptiveContext = AdaptiveContext(session)
 
   override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = {
     withTransformRules(transformRules(outputsColumnar)).apply(plan)

diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala
@@ -33,8 +33,8 @@ sealed trait AdaptiveContext {
 }
 
 object AdaptiveContext {
-  def apply(session: SparkSession, aqeStackTraceIndex: Int): AdaptiveContext =
-    new AdaptiveContextImpl(session, aqeStackTraceIndex)
+  def apply(session: SparkSession): AdaptiveContext =
+    new AdaptiveContextImpl(session)
 
   private val GLUTEN_IS_ADAPTIVE_CONTEXT = "gluten.isAdaptiveContext"
 
@@ -44,8 +44,7 @@ object AdaptiveContext {
   private val localIsAdaptiveContextFlags: ThreadLocal[ListBuffer[Boolean]] =
     ThreadLocal.withInitial(() => ListBuffer.empty[Boolean])
 
-  private class AdaptiveContextImpl(session: SparkSession, aqeStackTraceIndex: Int)
-    extends AdaptiveContext {
+  private class AdaptiveContextImpl(session: SparkSession) extends AdaptiveContext {
     // Just for test use.
     override def enableAdaptiveContext(): Unit = {
       session.sparkContext.setLocalProperty(GLUTEN_IS_ADAPTIVE_CONTEXT, "true")
@@ -59,19 +58,13 @@ object AdaptiveContext {
 
     override def setAdaptiveContext(): Unit = {
       val traceElements = Thread.currentThread.getStackTrace
-      assert(
-        traceElements.length > aqeStackTraceIndex,
-        s"The number of stack trace elements is expected to be more than $aqeStackTraceIndex")
       // ApplyColumnarRulesAndInsertTransitions is called by either QueryExecution or
       // AdaptiveSparkPlanExec. So by checking the stack trace, we can know whether
-      // columnar rule will be applied in adaptive execution context. This part of code
-      // needs to be carefully checked when supporting higher versions of spark to make
-      // sure the calling stack has not been changed.
+      // columnar rule will be applied in adaptive execution context.
       localIsAdaptiveContextFlags
         .get()
         .prepend(
-          traceElements(aqeStackTraceIndex).getClassName
-            .equals(AdaptiveSparkPlanExec.getClass.getName))
+          traceElements.exists(_.getClassName.equals(AdaptiveSparkPlanExec.getClass.getName)))
     }
 
     override def resetAdaptiveContext(): Unit =

diff --git a/gluten-ut/pom.xml b/gluten-ut/pom.xml
@@ -158,7 +158,7 @@
     <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>
-      <version>2.11.0</version>
+      <version>2.14.0</version>
       <scope>provided</scope>
     </dependency>
   </dependencies>