From 0bd6584eab916e79028c6275ae72e4cb3a0364a8 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 25 Nov 2024 14:04:45 +0800 Subject: [PATCH] [VL] RAS: Remove alternative constraint sets passing to RAS planner (#8033) --- .../enumerated/EnumeratedTransform.scala | 11 +----- .../scala/org/apache/gluten/ras/Ras.scala | 17 +++------ .../org/apache/gluten/ras/RasPlanner.scala | 10 ++---- .../org/apache/gluten/ras/dp/DpPlanner.scala | 15 ++------ .../ras/exaustive/ExhaustivePlanner.scala | 10 ++---- .../org/apache/gluten/ras/PropertySuite.scala | 35 ------------------- 6 files changed, 13 insertions(+), 85 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala index 43c5f10bbde3..fad0ae386c04 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala @@ -21,7 +21,6 @@ import org.apache.gluten.exception.GlutenException import org.apache.gluten.extension.columnar.ColumnarRuleApplier.ColumnarRuleCall import org.apache.gluten.extension.columnar.enumerated.planner.GlutenOptimization import org.apache.gluten.extension.columnar.enumerated.planner.property.Conv -import org.apache.gluten.extension.columnar.transition.ConventionReq import org.apache.gluten.extension.injector.Injector import org.apache.gluten.extension.util.AdaptiveContext import org.apache.gluten.logging.LogLevelUtil @@ -59,17 +58,9 @@ case class EnumeratedTransform(costModel: CostModel[SparkPlan], rules: Seq[RasRu private val reqConvention = Conv.any - private val altConventions = { - val rowBased: Conv = Conv.req(ConventionReq.row) - val backendBatchBased: Conv = Conv.req(ConventionReq.backendBatch) - Seq(rowBased, backendBatchBased) - } - override def apply(plan: SparkPlan): SparkPlan = { val constraintSet = PropertySet(List(reqConvention)) - val altConstraintSets = - altConventions.map(altConv => PropertySet(List(altConv))) - val planner = optimization.newPlanner(plan, constraintSet, altConstraintSets) + val planner = optimization.newPlanner(plan, constraintSet) val out = planner.plan() out } diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/Ras.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/Ras.scala index f705a2901be9..6cf15b0e8d6c 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/Ras.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/Ras.scala @@ -26,10 +26,7 @@ import scala.collection.mutable * https://github.com/apache/incubator-gluten/issues/5057. */ trait Optimization[T <: AnyRef] { - def newPlanner( - plan: T, - constraintSet: PropertySet[T], - altConstraintSets: Seq[PropertySet[T]]): RasPlanner[T] + def newPlanner(plan: T, constraintSet: PropertySet[T]): RasPlanner[T] def anyPropSet(): PropertySet[T] def withNewConfig(confFunc: RasConfig => RasConfig): Optimization[T] } @@ -47,10 +44,7 @@ object Optimization { implicit class OptimizationImplicits[T <: AnyRef](opt: Optimization[T]) { def newPlanner(plan: T): RasPlanner[T] = { - opt.newPlanner(plan, opt.anyPropSet(), List.empty) - } - def newPlanner(plan: T, constraintSet: PropertySet[T]): RasPlanner[T] = { - opt.newPlanner(plan, constraintSet, List.empty) + opt.newPlanner(plan, opt.anyPropSet()) } } } @@ -113,11 +107,8 @@ class Ras[T <: AnyRef] private ( } } - override def newPlanner( - plan: T, - constraintSet: PropertySet[T], - altConstraintSets: Seq[PropertySet[T]]): RasPlanner[T] = { - RasPlanner(this, altConstraintSets, constraintSet, plan) + override def newPlanner(plan: T, constraintSet: PropertySet[T]): RasPlanner[T] = { + RasPlanner(this, constraintSet, plan) } override def anyPropSet(): PropertySet[T] = propertySetFactory().any() diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/RasPlanner.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/RasPlanner.scala index 327b980f38ec..626d2b1caf9c 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/RasPlanner.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/RasPlanner.scala @@ -32,16 +32,12 @@ trait RasPlanner[T <: AnyRef] { } object RasPlanner { - def apply[T <: AnyRef]( - ras: Ras[T], - altConstraintSets: Seq[PropertySet[T]], - constraintSet: PropertySet[T], - plan: T): RasPlanner[T] = { + def apply[T <: AnyRef](ras: Ras[T], constraintSet: PropertySet[T], plan: T): RasPlanner[T] = { ras.config.plannerType match { case PlannerType.Exhaustive => - ExhaustivePlanner(ras, altConstraintSets, constraintSet, plan) + ExhaustivePlanner(ras, constraintSet, plan) case PlannerType.Dp => - DpPlanner(ras, altConstraintSets, constraintSet, plan) + DpPlanner(ras, constraintSet, plan) } } } diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/dp/DpPlanner.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/dp/DpPlanner.scala index 3f2590dff8a4..2b601720bfd5 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/dp/DpPlanner.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/dp/DpPlanner.scala @@ -26,11 +26,7 @@ import org.apache.gluten.ras.property.PropertySet import org.apache.gluten.ras.rule.{EnforcerRuleSet, RuleApplier, Shape} // TODO: Branch and bound pruning. -private class DpPlanner[T <: AnyRef] private ( - ras: Ras[T], - altConstraintSets: Seq[PropertySet[T]], - constraintSet: PropertySet[T], - plan: T) +private class DpPlanner[T <: AnyRef] private (ras: Ras[T], constraintSet: PropertySet[T], plan: T) extends RasPlanner[T] { import DpPlanner._ @@ -43,7 +39,6 @@ private class DpPlanner[T <: AnyRef] private ( } private lazy val best: (Best[T], KnownCostPath[T]) = { - altConstraintSets.foreach(propSet => memo.memorize(plan, propSet)) val groupId = rootGroupId val memoTable = memo.table() val best = findBest(memoTable, groupId) @@ -70,12 +65,8 @@ private class DpPlanner[T <: AnyRef] private ( } object DpPlanner { - def apply[T <: AnyRef]( - ras: Ras[T], - altConstraintSets: Seq[PropertySet[T]], - constraintSet: PropertySet[T], - plan: T): RasPlanner[T] = { - new DpPlanner(ras, altConstraintSets: Seq[PropertySet[T]], constraintSet, plan) + def apply[T <: AnyRef](ras: Ras[T], constraintSet: PropertySet[T], plan: T): RasPlanner[T] = { + new DpPlanner(ras, constraintSet, plan) } // Visited flag. diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/exaustive/ExhaustivePlanner.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/exaustive/ExhaustivePlanner.scala index c4d3e4881c43..f0cb42cf662a 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/exaustive/ExhaustivePlanner.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/exaustive/ExhaustivePlanner.scala @@ -27,7 +27,6 @@ import org.apache.gluten.ras.rule.{EnforcerRuleSet, RuleApplier, Shape} private class ExhaustivePlanner[T <: AnyRef] private ( ras: Ras[T], - altConstraintSets: Seq[PropertySet[T]], constraintSet: PropertySet[T], plan: T) extends RasPlanner[T] { @@ -40,7 +39,6 @@ private class ExhaustivePlanner[T <: AnyRef] private ( } private lazy val best: (Best[T], KnownCostPath[T]) = { - altConstraintSets.foreach(propSet => memo.memorize(plan, propSet)) val groupId = rootGroupId explore() val memoState = memo.newState() @@ -72,12 +70,8 @@ private class ExhaustivePlanner[T <: AnyRef] private ( } object ExhaustivePlanner { - def apply[T <: AnyRef]( - ras: Ras[T], - altConstraintSets: Seq[PropertySet[T]], - constraintSet: PropertySet[T], - plan: T): RasPlanner[T] = { - new ExhaustivePlanner(ras, altConstraintSets, constraintSet, plan) + def apply[T <: AnyRef](ras: Ras[T], constraintSet: PropertySet[T], plan: T): RasPlanner[T] = { + new ExhaustivePlanner(ras, constraintSet, plan) } private class ExhaustiveExplorer[T <: AnyRef]( diff --git a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/PropertySuite.scala b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/PropertySuite.scala index eb4babe069e5..06bb806f7d6b 100644 --- a/gluten-ras/common/src/test/scala/org/apache/gluten/ras/PropertySuite.scala +++ b/gluten-ras/common/src/test/scala/org/apache/gluten/ras/PropertySuite.scala @@ -250,41 +250,6 @@ abstract class PropertySuite extends AnyFunSuite { assert(out == TypedUnary(TypeA, 8, PassNodeType(5, TypedLeaf(TypeA, 10)))) } - test(s"Property convert - (A, B), alternative conventions") { - object ConvertEnforcerAndTypeAToTypeB extends RasRule[TestNode] { - override def shift(node: TestNode): Iterable[TestNode] = node match { - case TypeEnforcer(TypeB, _, TypedBinary(TypeA, 5, left, right)) => - List(TypedBinary(TypeB, 0, left, right)) - case _ => List.empty - } - override def shape(): Shape[TestNode] = Shapes.fixedHeight(2) - } - - val ras = - Ras[TestNode]( - PlanModelImpl, - CostModelImpl, - MetadataModelImpl, - propertyModel(zeroDepth), - ExplainImpl, - RasRule.Factory.reuse(List(ConvertEnforcerAndTypeAToTypeB))) - .withNewConfig(_ => conf) - val plan = - TypedBinary(TypeA, 5, TypedUnary(TypeA, 10, TypedLeaf(TypeA, 10)), TypedLeaf(TypeA, 10)) - val planner = ras.newPlanner( - plan, - PropertySet(Seq(TypeAny)), - List(PropertySet(Seq(TypeB)), PropertySet(Seq(TypeC)))) - val out = planner.plan() - assert( - out == TypedBinary( - TypeB, - 0, - TypeEnforcer(TypeB, 1, TypedUnary(TypeA, 10, TypedLeaf(TypeA, 10))), - TypeEnforcer(TypeB, 1, TypedLeaf(TypeA, 10)))) - assert(planner.newState().memoState().allGroups().size == 9) - } - test(s"Property convert - (A, B), Unary only has TypeA") { object ReplaceNonUnaryByTypeBRule extends RasRule[TestNode] { override def shift(node: TestNode): Iterable[TestNode] = {