From 7ca509f9b2c258473f92449cc02d6d4d71202cc9 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 3 Sep 2024 14:26:45 +0800 Subject: [PATCH] [GLUTEN-7031] Move task lifecycle management / memory consumer facilities to gluten-core (#7088) --- .../apache/gluten/memory/CHThreadGroup.java | 4 ++-- .../GlutenClickHouseTPCDSMetricsSuite.scala | 2 +- .../GlutenClickHouseTPCHMetricsSuite.scala | 2 +- .../velox/VeloxTransformerApi.scala | 2 +- .../backendsapi/velox/VeloxValidatorApi.scala | 2 +- .../v2/ArrowCSVPartitionReaderFactory.scala | 3 ++- .../gluten/datasource/v2/ArrowCSVTable.scala | 2 +- .../execution/RowToVeloxColumnarExec.scala | 2 +- .../VeloxBloomFilterMightContain.scala | 2 +- .../aggregate/VeloxBloomFilterAggregate.scala | 2 +- .../spark/sql/execution/BroadcastUtils.scala | 2 +- .../columnarbatch/ColumnarBatchTest.java | 2 +- .../gluten/utils/VeloxBloomFilterTest.java | 2 +- .../vectorized/ArrowColumnVectorTest.java | 2 +- cpp/core/CMakeLists.txt | 8 +++---- ...VeloxCelebornColumnarBatchSerializer.scala | 2 +- gluten-core/pom.xml | 22 +++++++++++++++++++ .../gluten/memory/MemoryUsageRecorder.java | 0 .../memory/MemoryUsageStatsBuilder.java | 0 .../memory/SimpleMemoryUsageRecorder.java | 0 .../DynamicOffHeapSizingMemoryTarget.java | 0 .../memory/memtarget/KnownNameAndStats.java | 0 .../memory/memtarget/LoggingMemoryTarget.java | 0 .../gluten/memory/memtarget/MemoryTarget.java | 0 .../memory/memtarget/MemoryTargetUtil.java | 0 .../memory/memtarget/MemoryTargetVisitor.java | 0 .../memory/memtarget/MemoryTargets.java | 0 .../memory/memtarget/NoopMemoryTarget.java | 0 .../gluten/memory/memtarget/OverAcquire.java | 0 .../gluten/memory/memtarget/Spiller.java | 0 .../gluten/memory/memtarget/Spillers.java | 0 .../memtarget/ThrowOnOomMemoryTarget.java | 2 +- .../memory/memtarget/TreeMemoryTarget.java | 0 .../memory/memtarget/TreeMemoryTargets.java | 0 .../spark/RegularMemoryConsumer.java | 2 +- .../memtarget/spark/TreeMemoryConsumer.java | 0 .../memtarget/spark/TreeMemoryConsumers.java | 0 .../apache/gluten/task}/TaskListener.scala | 2 +- .../org/apache/gluten/proto/config.proto | 0 .../org/apache/gluten/proto/memory.proto | 0 .../apache/spark/memory/SparkMemoryUtil.scala | 2 +- .../org/apache/spark/task}/TaskResource.scala | 2 +- .../apache/spark/task}/TaskResources.scala | 20 +++++++++-------- .../gluten/columnarbatch/IndicatorVector.java | 2 +- .../columnarbatch/IndicatorVectorPool.java | 2 +- .../arrow/alloc/ArrowBufferAllocators.java | 4 ++-- .../arrow/pool/ArrowNativeMemoryPool.java | 4 ++-- .../memory/listener/ReservationListeners.java | 2 +- .../metrics/HashAggregateMetricsUpdater.scala | 2 +- .../gluten/metrics/JoinMetricsUpdater.scala | 2 +- .../gluten/metrics/SortMetricsUpdater.scala | 2 +- .../org/apache/gluten/runtime/Runtime.scala | 2 +- .../org/apache/gluten/runtime/Runtimes.scala | 2 +- .../vectorized/ColumnarBatchSerializer.scala | 2 +- .../execution/ColumnarBuildSideRelation.scala | 2 +- .../MassiveMemoryAllocationSuite.scala | 2 +- .../org/apache/gluten/GlutenPlugin.scala | 5 +++-- .../gluten/planner/plan/GlutenPlanModel.scala | 3 ++- .../gluten/utils/iterator/IteratorsV1.scala | 2 +- .../spark/TreeMemoryConsumerTest.java | 2 +- .../gluten/utils/TaskResourceSuite.scala | 3 ++- .../gluten/utils/iterator/IteratorSuite.scala | 2 +- .../utils/iterator/IteratorBenchmark.scala | 2 +- 63 files changed, 85 insertions(+), 57 deletions(-) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/MemoryUsageStatsBuilder.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/SimpleMemoryUsageRecorder.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/KnownNameAndStats.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/LoggingMemoryTarget.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/MemoryTarget.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetUtil.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/NoopMemoryTarget.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/OverAcquire.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java (99%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java (99%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java (100%) rename {gluten-substrait => gluten-core}/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java (100%) rename {gluten-substrait/src/main/scala/org/apache/gluten/utils => gluten-core/src/main/java/org/apache/gluten/task}/TaskListener.scala (96%) rename {gluten-substrait => gluten-core}/src/main/resources/org/apache/gluten/proto/config.proto (100%) rename {gluten-substrait => gluten-core}/src/main/resources/org/apache/gluten/proto/memory.proto (100%) rename {gluten-substrait => gluten-core}/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala (97%) rename {gluten-substrait/src/main/scala/org/apache/spark/util => gluten-core/src/main/scala/org/apache/spark/task}/TaskResource.scala (97%) rename {gluten-substrait/src/main/scala/org/apache/spark/util => gluten-core/src/main/scala/org/apache/spark/task}/TaskResources.scala (95%) diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/memory/CHThreadGroup.java b/backends-clickhouse/src/main/java/org/apache/gluten/memory/CHThreadGroup.java index a06c552a9f6b..119dc61893d9 100644 --- a/backends-clickhouse/src/main/java/org/apache/gluten/memory/CHThreadGroup.java +++ b/backends-clickhouse/src/main/java/org/apache/gluten/memory/CHThreadGroup.java @@ -17,8 +17,8 @@ package org.apache.gluten.memory; import org.apache.spark.TaskContext; -import org.apache.spark.util.TaskResource; -import org.apache.spark.util.TaskResources; +import org.apache.spark.task.TaskResource; +import org.apache.spark.task.TaskResources; public class CHThreadGroup implements TaskResource { diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCDSMetricsSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCDSMetricsSuite.scala index 79017eab0d51..3e3899b0e115 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCDSMetricsSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCDSMetricsSuite.scala @@ -23,7 +23,7 @@ import org.apache.gluten.vectorized.GeneralInIterator import org.apache.spark.SparkConf import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.execution.InputIteratorTransformer -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources import scala.collection.JavaConverters._ diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala index 509967125a64..a5d81b781b32 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseTPCHMetricsSuite.scala @@ -23,7 +23,7 @@ import org.apache.gluten.vectorized.GeneralInIterator import org.apache.spark.SparkConf import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.execution.InputIteratorTransformer -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources import scala.collection.JavaConverters._ diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala index 37d8acc7f2b7..7cc4f5e8100a 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} import org.apache.spark.sql.types._ -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources import org.apache.spark.util.collection.BitSet import com.google.protobuf.{Any, Message} diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala index 131951a96fc8..e15270f47304 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.plans.physical.Partitioning import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.types._ -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources class VeloxValidatorApi extends ValidatorApi { diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala index 4af5022a6252..c930cebebe69 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVPartitionReaderFactory.scala @@ -33,7 +33,8 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources.Filter import org.apache.spark.sql.types.{StructField, StructType} import org.apache.spark.sql.vectorized.ColumnarBatch -import org.apache.spark.util.{SerializableConfiguration, TaskResources} +import org.apache.spark.task.TaskResources +import org.apache.spark.util.SerializableConfiguration import org.apache.arrow.c.ArrowSchema import org.apache.arrow.vector.types.pojo.Schema diff --git a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala index 02485975e705..3eaf4e35fd21 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/datasource/v2/ArrowCSVTable.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.datasources.v2.FileTable import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources import org.apache.hadoop.fs.FileStatus diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala index aa30cc80d4db..542c325f095e 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala @@ -34,8 +34,8 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.sql.utils.SparkArrowUtil import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.spark.task.TaskResources import org.apache.spark.unsafe.Platform -import org.apache.spark.util.TaskResources import org.apache.arrow.c.ArrowSchema import org.apache.arrow.memory.ArrowBuf diff --git a/backends-velox/src/main/scala/org/apache/gluten/expression/VeloxBloomFilterMightContain.scala b/backends-velox/src/main/scala/org/apache/gluten/expression/VeloxBloomFilterMightContain.scala index 8c2a7e492a49..05e1e3eb481f 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/expression/VeloxBloomFilterMightContain.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/expression/VeloxBloomFilterMightContain.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression} import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.catalyst.expressions.codegen.Block.BlockHelper import org.apache.spark.sql.types.DataType -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources /** * Velox's bloom-filter implementation uses different algorithms internally comparing to vanilla diff --git a/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/VeloxBloomFilterAggregate.scala b/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/VeloxBloomFilterAggregate.scala index d22cc7023abf..976abb9e21fb 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/VeloxBloomFilterAggregate.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/expression/aggregate/VeloxBloomFilterAggregate.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.TypedImperativeAggreg import org.apache.spark.sql.catalyst.trees.TernaryLike import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.DataType -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources import org.apache.spark.util.sketch.BloomFilter /** diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala index 29a12f532bea..5bc6b7c56da5 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.plans.physical.{BroadcastMode, BroadcastPar import org.apache.spark.sql.execution.joins.{HashedRelation, HashedRelationBroadcastMode, LongHashedRelation} import org.apache.spark.sql.types.StructType import org.apache.spark.sql.vectorized.ColumnarBatch -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources import scala.collection.mutable.ArrayBuffer; diff --git a/backends-velox/src/test/java/org/apache/gluten/columnarbatch/ColumnarBatchTest.java b/backends-velox/src/test/java/org/apache/gluten/columnarbatch/ColumnarBatchTest.java index 54994ccd4836..819d35a100c3 100644 --- a/backends-velox/src/test/java/org/apache/gluten/columnarbatch/ColumnarBatchTest.java +++ b/backends-velox/src/test/java/org/apache/gluten/columnarbatch/ColumnarBatchTest.java @@ -24,7 +24,7 @@ import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.vectorized.ColumnarBatch; -import org.apache.spark.util.TaskResources$; +import org.apache.spark.task.TaskResources$; import org.junit.Assert; import org.junit.Test; diff --git a/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java b/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java index cf568b166582..86b14fa46496 100644 --- a/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java +++ b/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java @@ -18,7 +18,7 @@ import org.apache.gluten.test.VeloxBackendTestBase; -import org.apache.spark.util.TaskResources$; +import org.apache.spark.task.TaskResources$; import org.apache.spark.util.sketch.BloomFilter; import org.apache.spark.util.sketch.IncompatibleMergeException; import org.junit.Assert; diff --git a/backends-velox/src/test/java/org/apache/gluten/vectorized/ArrowColumnVectorTest.java b/backends-velox/src/test/java/org/apache/gluten/vectorized/ArrowColumnVectorTest.java index 11330544df78..c7c13450ce88 100644 --- a/backends-velox/src/test/java/org/apache/gluten/vectorized/ArrowColumnVectorTest.java +++ b/backends-velox/src/test/java/org/apache/gluten/vectorized/ArrowColumnVectorTest.java @@ -19,7 +19,7 @@ import org.apache.spark.sql.execution.vectorized.MutableColumnarRow; import org.apache.spark.sql.types.Decimal; import org.apache.spark.sql.types.StructType; -import org.apache.spark.util.TaskResources$; +import org.apache.spark.task.TaskResources$; import org.junit.Assert; import org.junit.Test; diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index 9311b71ed83e..4c84d15f3a77 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -38,14 +38,14 @@ endif() set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) +set(GLUTEN_PROTO_SRC_DIR + ${GLUTEN_HOME}/gluten-core/src/main/resources/org/apache/gluten/proto) +message(STATUS "Set Gluten Proto Directory in ${GLUTEN_PROTO_SRC_DIR}") + set(SUBSTRAIT_PROTO_SRC_DIR ${GLUTEN_HOME}/gluten-substrait/src/main/resources/substrait/proto) message(STATUS "Set Substrait Proto Directory in ${SUBSTRAIT_PROTO_SRC_DIR}") -set(GLUTEN_PROTO_SRC_DIR - ${GLUTEN_HOME}/gluten-substrait/src/main/resources/org/apache/gluten/proto) -message(STATUS "Set Gluten Proto Directory in ${GLUTEN_PROTO_SRC_DIR}") - find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala index 696a3c3438b0..5355baed5b58 100644 --- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala +++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala @@ -32,7 +32,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.sql.utils.SparkSchemaUtil import org.apache.spark.sql.vectorized.ColumnarBatch -import org.apache.spark.util.{TaskResource, TaskResources} +import org.apache.spark.task.{TaskResource, TaskResources} import org.apache.arrow.c.ArrowSchema import org.apache.arrow.memory.BufferAllocator diff --git a/gluten-core/pom.xml b/gluten-core/pom.xml index 08f5de5801a0..448bc9ee2d13 100644 --- a/gluten-core/pom.xml +++ b/gluten-core/pom.xml @@ -128,6 +128,28 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + compile-gluten-proto + generate-sources + + compile + test-compile + + + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + + src/main/resources/org/apache/gluten/proto + false + + + + org.apache.maven.plugins maven-resources-plugin diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java b/gluten-core/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java rename to gluten-core/src/main/java/org/apache/gluten/memory/MemoryUsageRecorder.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/MemoryUsageStatsBuilder.java b/gluten-core/src/main/java/org/apache/gluten/memory/MemoryUsageStatsBuilder.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/MemoryUsageStatsBuilder.java rename to gluten-core/src/main/java/org/apache/gluten/memory/MemoryUsageStatsBuilder.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/SimpleMemoryUsageRecorder.java b/gluten-core/src/main/java/org/apache/gluten/memory/SimpleMemoryUsageRecorder.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/SimpleMemoryUsageRecorder.java rename to gluten-core/src/main/java/org/apache/gluten/memory/SimpleMemoryUsageRecorder.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/KnownNameAndStats.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/KnownNameAndStats.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/KnownNameAndStats.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/KnownNameAndStats.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/LoggingMemoryTarget.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/LoggingMemoryTarget.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/LoggingMemoryTarget.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/LoggingMemoryTarget.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTarget.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTarget.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTarget.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTarget.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetUtil.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetUtil.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetUtil.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetUtil.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargetVisitor.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/NoopMemoryTarget.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/NoopMemoryTarget.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/NoopMemoryTarget.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/NoopMemoryTarget.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/OverAcquire.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/OverAcquire.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/OverAcquire.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/OverAcquire.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java similarity index 99% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java index 74e0cbb8779b..67edb713a7ce 100644 --- a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java @@ -20,7 +20,7 @@ import org.apache.spark.memory.SparkMemoryUtil; import org.apache.spark.sql.internal.SQLConf; -import org.apache.spark.util.TaskResources; +import org.apache.spark.task.TaskResources; import org.apache.spark.util.Utils; public class ThrowOnOomMemoryTarget implements MemoryTarget { diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java similarity index 99% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java index 76aa63aebb64..054b9ef106a7 100644 --- a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java @@ -30,7 +30,7 @@ import org.apache.spark.memory.MemoryConsumer; import org.apache.spark.memory.MemoryMode; import org.apache.spark.memory.TaskMemoryManager; -import org.apache.spark.util.TaskResources; +import org.apache.spark.task.TaskResources; import java.util.Map; import java.util.stream.Collectors; diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java diff --git a/gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java similarity index 100% rename from gluten-substrait/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java rename to gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/utils/TaskListener.scala b/gluten-core/src/main/java/org/apache/gluten/task/TaskListener.scala similarity index 96% rename from gluten-substrait/src/main/scala/org/apache/gluten/utils/TaskListener.scala rename to gluten-core/src/main/java/org/apache/gluten/task/TaskListener.scala index 3ffb3206b900..d99c7e8856f7 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/utils/TaskListener.scala +++ b/gluten-core/src/main/java/org/apache/gluten/task/TaskListener.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.utils +package org.apache.gluten.task import org.apache.spark.TaskFailedReason diff --git a/gluten-substrait/src/main/resources/org/apache/gluten/proto/config.proto b/gluten-core/src/main/resources/org/apache/gluten/proto/config.proto similarity index 100% rename from gluten-substrait/src/main/resources/org/apache/gluten/proto/config.proto rename to gluten-core/src/main/resources/org/apache/gluten/proto/config.proto diff --git a/gluten-substrait/src/main/resources/org/apache/gluten/proto/memory.proto b/gluten-core/src/main/resources/org/apache/gluten/proto/memory.proto similarity index 100% rename from gluten-substrait/src/main/resources/org/apache/gluten/proto/memory.proto rename to gluten-core/src/main/resources/org/apache/gluten/proto/memory.proto diff --git a/gluten-substrait/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala b/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala similarity index 97% rename from gluten-substrait/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala rename to gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala index 6fd2d7a3ec10..d221fafce418 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala +++ b/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.memory -import org.apache.gluten.memory.memtarget.{DynamicOffHeapSizingMemoryTarget, KnownNameAndStats, LoggingMemoryTarget, MemoryTarget, MemoryTargetVisitor, NoopMemoryTarget, OverAcquire, ThrowOnOomMemoryTarget, TreeMemoryTargets} +import org.apache.gluten.memory.memtarget._ import org.apache.gluten.memory.memtarget.spark.{RegularMemoryConsumer, TreeMemoryConsumer} import org.apache.gluten.proto.MemoryUsageStats diff --git a/gluten-substrait/src/main/scala/org/apache/spark/util/TaskResource.scala b/gluten-core/src/main/scala/org/apache/spark/task/TaskResource.scala similarity index 97% rename from gluten-substrait/src/main/scala/org/apache/spark/util/TaskResource.scala rename to gluten-core/src/main/scala/org/apache/spark/task/TaskResource.scala index 8dfa2110b53b..ca9bbd9dd981 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/util/TaskResource.scala +++ b/gluten-core/src/main/scala/org/apache/spark/task/TaskResource.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.util +package org.apache.spark.task /** * Manages the lifecycle for a specific type of memory resource managed by Spark. See also diff --git a/gluten-substrait/src/main/scala/org/apache/spark/util/TaskResources.scala b/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala similarity index 95% rename from gluten-substrait/src/main/scala/org/apache/spark/util/TaskResources.scala rename to gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala index 2ab2a41a4f9b..b061aa332c74 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/util/TaskResources.scala +++ b/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala @@ -14,15 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.util +package org.apache.spark.task + +import org.apache.gluten.task.TaskListener import org.apache.spark.{TaskContext, TaskFailedReason, TaskKilledException, UnknownReason} import org.apache.spark.internal.Logging import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.util.{TaskCompletionListener, TaskFailureListener} import _root_.org.apache.gluten.memory.SimpleMemoryUsageRecorder import _root_.org.apache.gluten.sql.shims.SparkShimLoader -import _root_.org.apache.gluten.utils.TaskListener import java.util import java.util.{Collections, Properties, UUID} @@ -288,7 +290,7 @@ class TaskResourceRegistry extends Logging { } /** Release all managed resources according to priority and reversed order */ - private[util] def releaseAll(): Unit = lock { + private[task] def releaseAll(): Unit = lock { val table = new util.ArrayList(priorityToResourcesMapping.entrySet()) Collections.sort( table, @@ -310,7 +312,7 @@ class TaskResourceRegistry extends Logging { } /** Release single resource by ID */ - private[util] def releaseResource(id: String): Unit = lock { + private[task] def releaseResource(id: String): Unit = lock { if (!resources.containsKey(id)) { throw new IllegalArgumentException( String.format("TaskResource with ID %s is not registered", id)) @@ -328,7 +330,7 @@ class TaskResourceRegistry extends Logging { resources.remove(id) } - private[util] def addResourceIfNotRegistered[T <: TaskResource](id: String, factory: () => T): T = + private[task] def addResourceIfNotRegistered[T <: TaskResource](id: String, factory: () => T): T = lock { if (resources.containsKey(id)) { return resources.get(id).asInstanceOf[T] @@ -338,7 +340,7 @@ class TaskResourceRegistry extends Logging { resource } - private[util] def addResource[T <: TaskResource](id: String, resource: T): T = lock { + private[task] def addResource[T <: TaskResource](id: String, resource: T): T = lock { if (resources.containsKey(id)) { throw new IllegalArgumentException( String.format("TaskResource with ID %s is already registered", id)) @@ -347,11 +349,11 @@ class TaskResourceRegistry extends Logging { resource } - private[util] def isResourceRegistered(id: String): Boolean = lock { + private[task] def isResourceRegistered(id: String): Boolean = lock { resources.containsKey(id) } - private[util] def getResource[T <: TaskResource](id: String): T = lock { + private[task] def getResource[T <: TaskResource](id: String): T = lock { if (!resources.containsKey(id)) { throw new IllegalArgumentException( String.format("TaskResource with ID %s is not registered", id)) @@ -359,7 +361,7 @@ class TaskResourceRegistry extends Logging { resources.get(id).asInstanceOf[T] } - private[util] def getSharedUsage(): SimpleMemoryUsageRecorder = lock { + private[task] def getSharedUsage(): SimpleMemoryUsageRecorder = lock { sharedUsage } } diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java index 7fe87e95fa54..3f09a3619b3a 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.columnarbatch; -import org.apache.spark.util.TaskResources; +import org.apache.spark.task.TaskResources; import java.util.concurrent.atomic.AtomicLong; diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorPool.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorPool.java index 6e46742b564a..c122cc1cca48 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorPool.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorPool.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.columnarbatch; -import org.apache.spark.util.TaskResource; +import org.apache.spark.task.TaskResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java index f51852ab8d2b..4a57a0349916 100644 --- a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java +++ b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java @@ -23,8 +23,8 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.spark.memory.TaskMemoryManager; -import org.apache.spark.util.TaskResource; -import org.apache.spark.util.TaskResources; +import org.apache.spark.task.TaskResource; +import org.apache.spark.task.TaskResources; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java index 04a6e0002ade..a1f07c949eba 100644 --- a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java +++ b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/pool/ArrowNativeMemoryPool.java @@ -17,8 +17,8 @@ package org.apache.gluten.memory.arrow.pool; import org.apache.arrow.dataset.jni.NativeMemoryPool; -import org.apache.spark.util.TaskResource; -import org.apache.spark.util.TaskResources; +import org.apache.spark.task.TaskResource; +import org.apache.spark.task.TaskResources; import org.apache.spark.util.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java index db5ac8426df0..b221db13e375 100644 --- a/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java +++ b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java @@ -22,7 +22,7 @@ import org.apache.gluten.memory.memtarget.*; import org.apache.spark.memory.TaskMemoryManager; -import org.apache.spark.util.TaskResources; +import org.apache.spark.task.TaskResources; import java.util.Collections; import java.util.Map; diff --git a/gluten-data/src/main/scala/org/apache/gluten/metrics/HashAggregateMetricsUpdater.scala b/gluten-data/src/main/scala/org/apache/gluten/metrics/HashAggregateMetricsUpdater.scala index 5053cc2ba03d..1bf422b0a0f5 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/metrics/HashAggregateMetricsUpdater.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/metrics/HashAggregateMetricsUpdater.scala @@ -20,7 +20,7 @@ import org.apache.gluten.substrait.AggregationParams import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.utils.SparkMetricsUtil -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources trait HashAggregateMetricsUpdater extends MetricsUpdater { def updateAggregationMetrics( diff --git a/gluten-data/src/main/scala/org/apache/gluten/metrics/JoinMetricsUpdater.scala b/gluten-data/src/main/scala/org/apache/gluten/metrics/JoinMetricsUpdater.scala index fe1fa2ad6caf..9b8f03dd7f45 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/metrics/JoinMetricsUpdater.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/metrics/JoinMetricsUpdater.scala @@ -21,7 +21,7 @@ import org.apache.gluten.substrait.JoinParams import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.utils.SparkMetricsUtil -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources import java.util diff --git a/gluten-data/src/main/scala/org/apache/gluten/metrics/SortMetricsUpdater.scala b/gluten-data/src/main/scala/org/apache/gluten/metrics/SortMetricsUpdater.scala index 0ba48c62cdfe..cb01ae151035 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/metrics/SortMetricsUpdater.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/metrics/SortMetricsUpdater.scala @@ -18,7 +18,7 @@ package org.apache.gluten.metrics import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.utils.SparkMetricsUtil -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources class SortMetricsUpdater(val metrics: Map[String, SQLMetric]) extends MetricsUpdater { diff --git a/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala index 63db3e58e51f..b18d4044dd69 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala @@ -27,7 +27,7 @@ import org.apache.gluten.proto.MemoryUsageStats import org.apache.spark.memory.SparkMemoryUtil import org.apache.spark.sql.internal.{GlutenConfigUtil, SQLConf} -import org.apache.spark.util.TaskResource +import org.apache.spark.task.TaskResource import org.slf4j.LoggerFactory diff --git a/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtimes.scala b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtimes.scala index 6d5e11afeff5..bfb2465b12b9 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtimes.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtimes.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.runtime -import org.apache.spark.util.{TaskResource, TaskResources} +import org.apache.spark.task.{TaskResource, TaskResources} object Runtimes { diff --git a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala index 0bd78cb92c5c..c9f0d06f5d06 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.sql.utils.SparkSchemaUtil import org.apache.spark.sql.vectorized.ColumnarBatch -import org.apache.spark.util.{TaskResource, TaskResources} +import org.apache.spark.task.{TaskResource, TaskResources} import org.apache.arrow.c.ArrowSchema import org.apache.arrow.memory.BufferAllocator diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala index cb65dbca4db0..09805cc2f324 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.execution.joins.BuildSideRelation import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.utils.SparkArrowUtil import org.apache.spark.sql.vectorized.ColumnarBatch -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources import org.apache.arrow.c.ArrowSchema diff --git a/gluten-data/src/test/scala/org/apache/gluten/execution/MassiveMemoryAllocationSuite.scala b/gluten-data/src/test/scala/org/apache/gluten/execution/MassiveMemoryAllocationSuite.scala index ebfa0e6123fd..f53c99e5e035 100644 --- a/gluten-data/src/test/scala/org/apache/gluten/execution/MassiveMemoryAllocationSuite.scala +++ b/gluten-data/src/test/scala/org/apache/gluten/execution/MassiveMemoryAllocationSuite.scala @@ -22,7 +22,7 @@ import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers} import org.apache.spark.SparkFunSuite import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources import java.util.concurrent.{Callable, Executors, TimeUnit} import java.util.concurrent.atomic.AtomicLong diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-substrait/src/main/scala/org/apache/gluten/GlutenPlugin.scala index d3179f6bbd32..66bab72bf577 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -21,8 +21,8 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.events.GlutenBuildInfoEvent import org.apache.gluten.exception.GlutenException import org.apache.gluten.extension.GlutenSessionExtensions.{GLUTEN_SESSION_EXTENSION_NAME, SPARK_SESSION_EXTS_KEY} +import org.apache.gluten.task.TaskListener import org.apache.gluten.test.TestStats -import org.apache.gluten.utils.TaskListener import org.apache.spark.{HdfsConfGenerator, SparkConf, SparkContext, TaskFailedReason} import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin, PluginContext, SparkPlugin} @@ -31,7 +31,8 @@ import org.apache.spark.listener.GlutenListenerFactory import org.apache.spark.network.util.JavaUtils import org.apache.spark.sql.execution.ui.GlutenEventUtils import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.util.{SparkResourceUtil, TaskResources} +import org.apache.spark.task.TaskResources +import org.apache.spark.util.SparkResourceUtil import java.util import java.util.Collections diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala b/gluten-substrait/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala index d981de8046a9..5df4f572c398 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/planner/plan/GlutenPlanModel.scala @@ -28,7 +28,8 @@ import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.execution.{ColumnarToRowExec, LeafExecNode, SparkPlan} -import org.apache.spark.util.{SparkTaskUtil, TaskResources} +import org.apache.spark.task.TaskResources +import org.apache.spark.util.SparkTaskUtil import java.util.{Objects, Properties} diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/utils/iterator/IteratorsV1.scala b/gluten-substrait/src/main/scala/org/apache/gluten/utils/iterator/IteratorsV1.scala index 3e9248c44458..1090c6944ae5 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/utils/iterator/IteratorsV1.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/utils/iterator/IteratorsV1.scala @@ -19,7 +19,7 @@ package org.apache.gluten.utils.iterator import org.apache.gluten.utils.iterator.Iterators.WrapperBuilder import org.apache.spark.{InterruptibleIterator, TaskContext} -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean diff --git a/gluten-substrait/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java b/gluten-substrait/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java index bbc43ba5dea9..befe449186e7 100644 --- a/gluten-substrait/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java +++ b/gluten-substrait/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java @@ -24,7 +24,7 @@ import org.apache.spark.TaskContext; import org.apache.spark.sql.internal.SQLConf; -import org.apache.spark.util.TaskResources$; +import org.apache.spark.task.TaskResources$; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/gluten-substrait/src/test/scala/org/apache/gluten/utils/TaskResourceSuite.scala b/gluten-substrait/src/test/scala/org/apache/gluten/utils/TaskResourceSuite.scala index 0c91b4faf004..47ffc8812d1c 100644 --- a/gluten-substrait/src/test/scala/org/apache/gluten/utils/TaskResourceSuite.scala +++ b/gluten-substrait/src/test/scala/org/apache/gluten/utils/TaskResourceSuite.scala @@ -19,7 +19,8 @@ package org.apache.gluten.utils import org.apache.spark.memory.{MemoryConsumer, MemoryMode} import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.util.{SparkTaskUtil, TaskResource, TaskResources} +import org.apache.spark.task.{TaskResource, TaskResources} +import org.apache.spark.util.SparkTaskUtil import org.scalatest.funsuite.AnyFunSuite diff --git a/gluten-substrait/src/test/scala/org/apache/gluten/utils/iterator/IteratorSuite.scala b/gluten-substrait/src/test/scala/org/apache/gluten/utils/iterator/IteratorSuite.scala index 1a84d671922d..9a70f235a339 100644 --- a/gluten-substrait/src/test/scala/org/apache/gluten/utils/iterator/IteratorSuite.scala +++ b/gluten-substrait/src/test/scala/org/apache/gluten/utils/iterator/IteratorSuite.scala @@ -18,7 +18,7 @@ package org.apache.gluten.utils.iterator import org.apache.gluten.utils.iterator.Iterators.{V1, WrapperBuilder} -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources import org.scalatest.funsuite.AnyFunSuite diff --git a/gluten-substrait/src/test/scala/org/apache/spark/utils/iterator/IteratorBenchmark.scala b/gluten-substrait/src/test/scala/org/apache/spark/utils/iterator/IteratorBenchmark.scala index aa69f309aac8..31f4848f3863 100644 --- a/gluten-substrait/src/test/scala/org/apache/spark/utils/iterator/IteratorBenchmark.scala +++ b/gluten-substrait/src/test/scala/org/apache/spark/utils/iterator/IteratorBenchmark.scala @@ -20,7 +20,7 @@ import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.utils.iterator.Iterators.V1 import org.apache.spark.benchmark.{Benchmark, BenchmarkBase} -import org.apache.spark.util.TaskResources +import org.apache.spark.task.TaskResources object IteratorBenchmark extends BenchmarkBase {