[GLUTEN-7031] Move iterator wrappers to gluten-core (#7095)

apache · Sep 4, 2024 · 51e6793 · 51e6793
1 parent 44634ad
commit 51e6793
Show file tree

Hide file tree

Showing 19 changed files with 25 additions and 42 deletions.
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala
@@ -19,13 +19,13 @@ package org.apache.gluten.backendsapi.velox
 import org.apache.gluten.GlutenNumaBindingInfo
 import org.apache.gluten.backendsapi.IteratorApi
 import org.apache.gluten.execution._
+import org.apache.gluten.iterator.Iterators
 import org.apache.gluten.metrics.IMetrics
 import org.apache.gluten.sql.shims.SparkShimLoader
 import org.apache.gluten.substrait.plan.PlanNode
 import org.apache.gluten.substrait.rel.{LocalFilesBuilder, LocalFilesNode, SplitInfo}
 import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat
 import org.apache.gluten.utils._
-import org.apache.gluten.utils.iterator.Iterators
 import org.apache.gluten.vectorized._
 
 import org.apache.spark.{SparkConf, TaskContext}

diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/ArrowCSVFileFormat.scala
@@ -19,10 +19,10 @@ package org.apache.gluten.datasource
 import org.apache.gluten.columnarbatch.ColumnarBatches
 import org.apache.gluten.exception.SchemaMismatchException
 import org.apache.gluten.execution.RowToVeloxColumnarExec
+import org.apache.gluten.iterator.Iterators
 import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
 import org.apache.gluten.memory.arrow.pool.ArrowNativeMemoryPool
 import org.apache.gluten.utils.ArrowUtil
-import org.apache.gluten.utils.iterator.Iterators
 import org.apache.gluten.vectorized.ArrowWritableColumnVector
 
 import org.apache.spark.TaskContext

diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
@@ -18,10 +18,10 @@ package org.apache.gluten.execution
 
 import org.apache.gluten.GlutenConfig
 import org.apache.gluten.columnarbatch.ColumnarBatches
+import org.apache.gluten.iterator.Iterators
 import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
 import org.apache.gluten.runtime.Runtimes
 import org.apache.gluten.utils.ArrowAbiUtil
-import org.apache.gluten.utils.iterator.Iterators
 import org.apache.gluten.vectorized._
 
 import org.apache.spark.broadcast.Broadcast

diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxBroadcastBuildSideRDD.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxBroadcastBuildSideRDD.scala
@@ -16,7 +16,7 @@
  */
 package org.apache.gluten.execution
 
-import org.apache.gluten.utils.iterator.Iterators
+import org.apache.gluten.iterator.Iterators
 
 import org.apache.spark.{broadcast, SparkContext}
 import org.apache.spark.sql.execution.joins.BuildSideRelation

diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala
@@ -19,8 +19,8 @@ package org.apache.gluten.execution
 import org.apache.gluten.columnarbatch.ColumnarBatches
 import org.apache.gluten.exception.GlutenNotSupportException
 import org.apache.gluten.extension.ValidationResult
+import org.apache.gluten.iterator.Iterators
 import org.apache.gluten.runtime.Runtimes
-import org.apache.gluten.utils.iterator.Iterators
 import org.apache.gluten.vectorized.NativeColumnarToRowJniWrapper
 
 import org.apache.spark.broadcast.Broadcast

diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxResizeBatchesExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxResizeBatchesExec.scala
@@ -17,8 +17,8 @@
 package org.apache.gluten.execution
 
 import org.apache.gluten.extension.GlutenPlan
+import org.apache.gluten.iterator.Iterators
 import org.apache.gluten.utils.VeloxBatchResizer
-import org.apache.gluten.utils.iterator.Iterators
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow

diff --git a/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala b/backends-velox/src/main/scala/org/apache/spark/api/python/ColumnarArrowEvalPythonExec.scala
@@ -19,9 +19,9 @@ package org.apache.spark.api.python
 import org.apache.gluten.columnarbatch.ColumnarBatches
 import org.apache.gluten.exception.GlutenException
 import org.apache.gluten.extension.GlutenPlan
+import org.apache.gluten.iterator.Iterators
 import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
 import org.apache.gluten.utils.PullOutProjectHelper
-import org.apache.gluten.utils.iterator.Iterators
 import org.apache.gluten.vectorized.ArrowWritableColumnVector
 
 import org.apache.spark.{ContextAwareIterator, SparkEnv, TaskContext}

diff --git a/...s-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/...s-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
@@ -20,10 +20,10 @@ import org.apache.gluten.GlutenConfig
 import org.apache.gluten.backendsapi.BackendsApiManager
 import org.apache.gluten.columnarbatch.ColumnarBatches
 import org.apache.gluten.execution.{RowToVeloxColumnarExec, VeloxColumnarToRowExec}
+import org.apache.gluten.iterator.Iterators
 import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
 import org.apache.gluten.runtime.Runtimes
 import org.apache.gluten.utils.ArrowAbiUtil
-import org.apache.gluten.utils.iterator.Iterators
 import org.apache.gluten.vectorized.ColumnarBatchSerializerJniWrapper
 
 import org.apache.spark.internal.Logging

diff --git a/...che/gluten/utils/iterator/Iterators.scala → ...rg/apache/gluten/iterator/Iterators.scala b/...che/gluten/utils/iterator/Iterators.scala → ...rg/apache/gluten/iterator/Iterators.scala
@@ -14,12 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.utils.iterator
-
-import org.apache.gluten.utils.iterator.IteratorsV1.WrapperBuilderV1
+package org.apache.gluten.iterator
 
 import org.apache.spark.TaskContext
 
+import IteratorsV1.WrapperBuilderV1
+
 /**
  * Utility class to provide iterator wrappers for non-trivial use cases. E.g. iterators that manage
  * payload's lifecycle.

diff --git a/...e/gluten/utils/iterator/IteratorsV1.scala → .../apache/gluten/iterator/IteratorsV1.scala b/...e/gluten/utils/iterator/IteratorsV1.scala → .../apache/gluten/iterator/IteratorsV1.scala
@@ -14,13 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.utils.iterator
-
-import org.apache.gluten.utils.iterator.Iterators.WrapperBuilder
+package org.apache.gluten.iterator
 
 import org.apache.spark.{InterruptibleIterator, TaskContext}
 import org.apache.spark.task.TaskResources
 
+import Iterators.WrapperBuilder
+
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicBoolean
 

diff --git a/...org/apache/spark/util/SparkTaskUtil.scala → ...org/apache/spark/task/SparkTaskUtil.scala b/...org/apache/spark/util/SparkTaskUtil.scala → ...org/apache/spark/task/SparkTaskUtil.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.util
+package org.apache.spark.task
 
 import org.apache.spark.TaskContext
 import org.apache.spark.memory.TaskMemoryManager

diff --git a/...mtarget/spark/TreeMemoryConsumerTest.java → ...mtarget/spark/TreeMemoryConsumerTest.java b/...mtarget/spark/TreeMemoryConsumerTest.java → ...mtarget/spark/TreeMemoryConsumerTest.java
diff --git a/...gluten/utils/iterator/IteratorSuite.scala → ...pache/gluten/iterator/IteratorSuite.scala b/...gluten/utils/iterator/IteratorSuite.scala → ...pache/gluten/iterator/IteratorSuite.scala
@@ -14,9 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.utils.iterator
+package org.apache.gluten.iterator
 
-import org.apache.gluten.utils.iterator.Iterators.{V1, WrapperBuilder}
+import org.apache.gluten.iterator.Iterators.{V1, WrapperBuilder}
 
 import org.apache.spark.task.TaskResources
 

diff --git a/...ache/gluten/utils/TaskResourceSuite.scala → ...pache/gluten/task/TaskResourceSuite.scala b/...ache/gluten/utils/TaskResourceSuite.scala → ...pache/gluten/task/TaskResourceSuite.scala
@@ -14,13 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.utils
+package org.apache.gluten.task
 
 import org.apache.spark.memory.{MemoryConsumer, MemoryMode}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.task.{TaskResource, TaskResources}
-import org.apache.spark.util.SparkTaskUtil
+import org.apache.spark.task.{SparkTaskUtil, TaskResource, TaskResources}
 
 import org.scalatest.funsuite.AnyFunSuite
 

diff --git a/...rk/utils/iterator/IteratorBenchmark.scala → ...he/spark/iterator/IteratorBenchmark.scala b/...rk/utils/iterator/IteratorBenchmark.scala → ...he/spark/iterator/IteratorBenchmark.scala
@@ -14,10 +14,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.utils.iterator
+package org.apache.spark.iterator
 
-import org.apache.gluten.utils.iterator.Iterators
-import org.apache.gluten.utils.iterator.Iterators.V1
+import org.apache.gluten.iterator.Iterators
+import org.apache.gluten.iterator.Iterators.V1
 
 import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
 import org.apache.spark.task.TaskResources

diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
@@ -17,11 +17,11 @@
 package org.apache.spark.sql.execution
 
 import org.apache.gluten.columnarbatch.ColumnarBatches
+import org.apache.gluten.iterator.Iterators
 import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
 import org.apache.gluten.runtime.Runtimes
 import org.apache.gluten.sql.shims.SparkShimLoader
 import org.apache.gluten.utils.ArrowAbiUtil
-import org.apache.gluten.utils.iterator.Iterators
 import org.apache.gluten.vectorized.{ColumnarBatchSerializerJniWrapper, NativeColumnarToRowJniWrapper}
 
 import org.apache.spark.sql.catalyst.InternalRow

diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala
@@ -17,9 +17,9 @@
 package org.apache.spark.sql.execution.utils
 
 import org.apache.gluten.columnarbatch.ColumnarBatches
+import org.apache.gluten.iterator.Iterators
 import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
 import org.apache.gluten.runtime.Runtimes
-import org.apache.gluten.utils.iterator.Iterators
 import org.apache.gluten.vectorized.{ArrowWritableColumnVector, NativeColumnarToRowInfo, NativeColumnarToRowJniWrapper, NativePartitioning}
 
 import org.apache.spark.{Partitioner, RangePartitioner, ShuffleDependency}

diff --git a/gluten-substrait/pom.xml b/gluten-substrait/pom.xml
@@ -296,21 +296,6 @@
               <clearOutputDirectory>true</clearOutputDirectory>
             </configuration>
           </execution>
-          <execution>
-            <id>compile-gluten-proto</id>
-            <phase>generate-sources</phase>
-            <goals>
-              <goal>compile</goal>
-              <goal>test-compile</goal>
-            </goals>
-            <configuration>
-              <protocArtifact>
-                com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
-              </protocArtifact>
-              <protoSourceRoot>src/main/resources/org/apache/gluten/proto</protoSourceRoot>
-              <clearOutputDirectory>false</clearOutputDirectory>
-            </configuration>
-          </execution>
         </executions>
       </plugin>
       <plugin>

diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala b/gluten-substrait/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala
@@ -28,8 +28,7 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.execution.{ColumnarToRowExec, LeafExecNode, SparkPlan}
-import org.apache.spark.task.TaskResources
-import org.apache.spark.util.SparkTaskUtil
+import org.apache.spark.task.{SparkTaskUtil, TaskResources}
 
 import java.util.{Objects, Properties}