[VL] Quick fix for Uniffle CI error (#5986)

apache · Jun 5, 2024 · 59aaa1c · 59aaa1c
1 parent 00dda1b
commit 59aaa1c
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 11 deletions.
diff --git a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/BaseMixin.java
@@ -75,7 +75,7 @@ public class BaseMixin {
   @CommandLine.Option(names = {"--shuffle-partitions"}, description = "Shuffle partition number", defaultValue = "100")
   private int shufflePartitions;
 
-  @CommandLine.Option(names = {"--scan-partitions"}, description = "Scan partition number. This is an approximate value, so the actual scan partition number might vary around this value", defaultValue = "100")
+  @CommandLine.Option(names = {"--scan-partitions"}, description = "Scan partition number. This is an approximate value, so the actual scan partition number might vary around this value. -1 for letting Spark choose an appropriate number.", defaultValue = "-1")
   private int scanPartitions;
 
   @CommandLine.Option(names = {"--extra-conf"}, description = "Extra Spark config entries applying to generated Spark session. E.g. --extra-conf=k1=v1 --extra-conf=k2=v2")

diff --git a/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala b/tools/gluten-it/common/src/main/scala/org/apache/gluten/integration/Suite.scala
@@ -103,16 +103,18 @@ abstract class Suite(
     sessionSwitcher.defaultConf().setWarningOnOverriding("spark.sql.codegen.wholeStage", "false")
   }
 
-  // Scan partition number.
-  sessionSwitcher
-    .defaultConf()
-    .setWarningOnOverriding("spark.sql.files.maxPartitionBytes", s"${ByteUnit.PiB.toBytes(1L)}")
-  sessionSwitcher
-    .defaultConf()
-    .setWarningOnOverriding("spark.sql.files.openCostInBytes", "0")
-  sessionSwitcher
-    .defaultConf()
-    .setWarningOnOverriding("spark.sql.files.minPartitionNum", s"${(scanPartitions - 1) max 1}")
+  if (scanPartitions != -1) {
+    // Scan partition number.
+    sessionSwitcher
+      .defaultConf()
+      .setWarningOnOverriding("spark.sql.files.maxPartitionBytes", s"${ByteUnit.PiB.toBytes(1L)}")
+    sessionSwitcher
+      .defaultConf()
+      .setWarningOnOverriding("spark.sql.files.openCostInBytes", "0")
+    sessionSwitcher
+      .defaultConf()
+      .setWarningOnOverriding("spark.sql.files.minPartitionNum", s"${(scanPartitions - 1) max 1}")
+  }
 
   extraSparkConf.toStream.foreach { kv =>
     sessionSwitcher.defaultConf().setWarningOnOverriding(kv._1, kv._2)