rss use row-based sort

apache · Aug 6, 2024 · 020eaf6 · 020eaf6
1 parent d746f0f
commit 020eaf6
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 5 deletions.
diff --git a/...leborn/common/src/main/scala/org/apache/spark/shuffle/CelebornColumnarShuffleWriter.scala b/...leborn/common/src/main/scala/org/apache/spark/shuffle/CelebornColumnarShuffleWriter.scala
@@ -73,7 +73,6 @@ abstract class CelebornColumnarShuffleWriter[K, V](
   protected val shuffleWriterType: String =
     celebornConf.shuffleWriterMode.name
       .toLowerCase(Locale.ROOT)
-      .replace(GlutenConfig.GLUTEN_SORT_SHUFFLE_WRITER, GlutenConfig.GLUTEN_RSS_SORT_SHUFFLE_WRITER)
 
   protected val celebornPartitionPusher = new CelebornPartitionPusher(
     shuffleId,

diff --git a/.../velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala b/.../velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
@@ -17,7 +17,6 @@
 package org.apache.spark.shuffle
 
 import org.apache.gluten.GlutenConfig
-import org.apache.gluten.GlutenConfig.{GLUTEN_RSS_SORT_SHUFFLE_WRITER, GLUTEN_SORT_SHUFFLE_WRITER}
 import org.apache.gluten.exec.Runtimes
 import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
 import org.apache.gluten.utils.ArrowAbiUtil
@@ -85,7 +84,6 @@ private class CelebornColumnarBatchSerializerInstance(
     val compressionCodecBackend =
       GlutenConfig.getConf.columnarShuffleCodecBackend.orNull
     val shuffleWriterType = GlutenConfig.getConf.celebornShuffleWriterType
-      .replace(GLUTEN_SORT_SHUFFLE_WRITER, GLUTEN_RSS_SORT_SHUFFLE_WRITER)
     val jniWrapper = ShuffleReaderJniWrapper.create(runtime)
     val batchSize = GlutenConfig.getConf.maxBatchSize
     val handle = jniWrapper

diff --git a/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala b/gluten-core/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala
@@ -217,8 +217,8 @@ object ColumnarShuffleExchangeExec extends Logging {
       conf.celebornShuffleWriterType == GlutenConfig.GLUTEN_SORT_SHUFFLE_WRITER
     partitioning != SinglePartition &&
     (partitioning.numPartitions >= GlutenConfig.getConf.columnarShuffleSortPartitionsThreshold ||
-      output.size >= GlutenConfig.getConf.columnarShuffleSortColumnsThreshold) ||
-    isCelebornSortBasedShuffle
+      output.size >= GlutenConfig.getConf.columnarShuffleSortColumnsThreshold ||
+      isCelebornSortBasedShuffle)
   }
 
   class DummyPairRDDWithPartitions(@transient private val sc: SparkContext, numPartitions: Int)