address comment and remove unnecessary log

apache · Jan 2, 2025 · efff600 · efff600
1 parent 7e20ba0
commit efff600
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 9 deletions.
diff --git a/...s-velox/src/main/scala/org/apache/spark/sql/execution/unsafe/UnsafeBytesBufferArray.scala b/...s-velox/src/main/scala/org/apache/spark/sql/execution/unsafe/UnsafeBytesBufferArray.scala
@@ -16,6 +16,7 @@
  */
 package org.apache.spark.sql.execution.unsafe
 
+import org.apache.spark.annotation.Experimental
 import org.apache.spark.internal.Logging
 import org.apache.spark.memory.{MemoryConsumer, MemoryMode, TaskMemoryManager}
 import org.apache.spark.unsafe.Platform
@@ -33,6 +34,7 @@ import org.apache.spark.unsafe.array.LongArray
  *   all bytesBuffer's length plus together
  */
 // scalastyle:off no.finalize
+@Experimental
 case class UnsafeBytesBufferArray(
     arraySize: Int,
     bytesBufferLengths: Array[Int],
@@ -99,7 +101,6 @@ case class UnsafeBytesBufferArray(
       return new Array[Byte](0)
     }
     val bytes = new Array[Byte](bytesBufferLengths(index))
-    log.warn(s"get bytesBuffer at index $index bytesBuffer length ${bytes.length}")
     Platform.copyMemory(
       longArray.getBaseObject,
       longArray.getBaseOffset + bytesBufferOffset(index),

diff --git a/...rc/main/scala/org/apache/spark/sql/execution/unsafe/UnsafeColumnarBuildSideRelation.scala b/...rc/main/scala/org/apache/spark/sql/execution/unsafe/UnsafeColumnarBuildSideRelation.scala
@@ -26,6 +26,7 @@ import org.apache.gluten.utils.ArrowAbiUtil
 import org.apache.gluten.vectorized.{ColumnarBatchSerializerJniWrapper, NativeColumnarToRowJniWrapper}
 
 import org.apache.spark.{SparkEnv, TaskContext}
+import org.apache.spark.annotation.Experimental
 import org.apache.spark.internal.Logging
 import org.apache.spark.memory.TaskMemoryManager
 import org.apache.spark.sql.catalyst.InternalRow
@@ -56,6 +57,7 @@ import scala.collection.JavaConverters.asScalaIteratorConverter
  * @param mode
  *   the broadcast mode.
  */
+@Experimental
 case class UnsafeColumnarBuildSideRelation(
     private var output: Seq[Attribute],
     private var batches: UnsafeBytesBufferArray,
@@ -135,7 +137,6 @@ case class UnsafeColumnarBuildSideRelation(
       val length = bytesBufferLengths(i)
       val tmpBuffer = new Array[Byte](length)
       in.read(tmpBuffer)
-      log.warn(s"readExternal $i--- $length")
       batches.putBytesBuffer(i, tmpBuffer)
     }
   }
@@ -169,7 +170,9 @@ case class UnsafeColumnarBuildSideRelation(
 
   override def deserialized: Iterator[ColumnarBatch] = {
     val runtime =
-      Runtimes.contextInstance(BackendsApiManager.getBackendName, "BuildSideRelation#transform")
+      Runtimes.contextInstance(
+        BackendsApiManager.getBackendName,
+        "UnsafeBuildSideRelation#deserialize")
     val jniWrapper = ColumnarBatchSerializerJniWrapper.create(runtime)
     val serializerHandle: Long = {
       val allocator = ArrowBufferAllocators.contextInstance()

diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala
@@ -331,15 +331,15 @@ class VeloxTPCHV1BhjSuite extends VeloxTPCHSuite {
   }
 }
 
-/** BroadcastBuildSideRelation use onheap. */
-class VeloxTPCHV1BhjOnheapSuite extends VeloxTPCHSuite {
-  override def subType(): String = "v1-bhj-onheap"
+/** BroadcastBuildSideRelation use off-heap. */
+class VeloxTPCHV1BhjOffheapSuite extends VeloxTPCHSuite {
+  override def subType(): String = "v1-bhj-off-heap"
 
   override protected def sparkConf: SparkConf = {
     super.sparkConf
       .set("spark.sql.sources.useV1SourceList", "parquet")
       .set("spark.sql.autoBroadcastJoinThreshold", "30M")
-      .set(GlutenConfig.VELOX_BROADCAST_BUILD_RELATION_USE_OFFHEAP.key, "false")
+      .set(GlutenConfig.VELOX_BROADCAST_BUILD_RELATION_USE_OFFHEAP.key, "true")
   }
 }
 

diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -2264,8 +2264,8 @@ object GlutenConfig {
   val VELOX_BROADCAST_BUILD_RELATION_USE_OFFHEAP =
     buildConf("spark.gluten.velox.offHeapBroadcastBuildRelation.enabled")
       .internal()
-      .doc("If enabled, broadcast build relation will use offheap memory. " +
+      .doc("Experimental: If enabled, broadcast build relation will use offheap memory. " +
         "Otherwise, broadcast build relation will use onheap memory.")
       .booleanConf
-      .createWithDefault(true)
+      .createWithDefault(false)
 }