From 9a6aa2c42845b4e2a0a8f2e93d7f0615640ad1b3 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 24 Sep 2024 08:19:55 +0800 Subject: [PATCH] [VL] Minor code cleanups (#7312) --- cpp/core/CMakeLists.txt | 3 +- cpp/core/benchmarks/CompressionBenchmark.cc | 2 +- cpp/core/compute/ProtobufUtils.cc | 2 +- cpp/core/compute/ResultIterator.h | 2 +- cpp/core/jni/JniCommon.h | 2 +- cpp/core/jni/JniError.h | 2 +- cpp/core/memory/ArrowMemoryPool.cc | 2 +- cpp/core/memory/ColumnarBatch.cc | 2 +- cpp/core/memory/ColumnarBatch.h | 2 +- cpp/core/memory/MemoryAllocator.cc | 2 +- cpp/core/operators/r2c/RowToColumnar.h | 2 +- cpp/core/shuffle/LocalPartitionWriter.h | 2 +- cpp/core/shuffle/Partitioner.cc | 2 +- cpp/core/shuffle/Partitioning.cc | 2 +- cpp/core/shuffle/Payload.cc | 2 +- cpp/core/shuffle/ShuffleReader.cc | 2 +- cpp/core/shuffle/ShuffleWriter.cc | 2 +- cpp/core/shuffle/Spill.h | 2 +- cpp/core/shuffle/rss/RssPartitionWriter.h | 2 +- cpp/core/utils/Compression.cc | 4 +- cpp/core/utils/{exception.h => Exception.h} | 0 cpp/core/utils/{macros.h => Macros.h} | 2 +- cpp/core/utils/{metrics.h => Metrics.h} | 0 cpp/core/utils/ResourceMap.h | 2 +- cpp/core/utils/StringUtil.cc | 2 +- .../utils/qpl/{qpl_codec.cc => QplCodec.cc} | 4 +- .../utils/qpl/{qpl_codec.h => QplCodec.h} | 2 +- .../qpl/{qpl_job_pool.cc => QplJobPool.cc} | 4 +- .../qpl/{qpl_job_pool.h => QplJobPool.h} | 2 +- .../benchmarks/ColumnarToRowBenchmark.cc | 2 +- cpp/velox/benchmarks/GenericBenchmark.cc | 2 +- cpp/velox/benchmarks/ParquetWriteBenchmark.cc | 2 +- cpp/velox/benchmarks/common/BenchmarkUtils.h | 2 +- .../benchmarks/common/ParquetReaderIterator.h | 2 +- cpp/velox/compute/VeloxBackend.cc | 4 +- cpp/velox/compute/WholeStageResultIterator.h | 2 +- cpp/velox/jni/JniUdf.cc | 2 +- cpp/velox/memory/ArrowMemory.h | 2 +- cpp/velox/memory/VeloxMemoryManager.cc | 2 +- .../serializer/VeloxColumnarToRowConverter.cc | 2 +- cpp/velox/shuffle/VeloxHashShuffleWriter.cc | 2 +- .../shuffle/VeloxRssSortShuffleWriter.cc | 2 +- cpp/velox/shuffle/VeloxShuffleReader.cc | 2 +- cpp/velox/substrait/VeloxToSubstraitPlan.cc | 2 +- cpp/velox/udf/UdfLoader.cc | 4 +- cpp/velox/utils/ConfigExtractor.cc | 2 +- cpp/velox/utils/HdfsUtils.cc | 2 +- cpp/velox/utils/VeloxBatchResizer.h | 2 +- cpp/velox/utils/tests/LocalRssClient.h | 2 +- cpp/velox/utils/tests/MemoryPoolUtils.h | 2 +- .../gluten/columnarbatch/ColumnarBatches.java | 8 ++-- .../apache/gluten/utils/ImplicitClass.scala | 38 ------------------- 52 files changed, 57 insertions(+), 96 deletions(-) rename cpp/core/utils/{exception.h => Exception.h} (100%) rename cpp/core/utils/{macros.h => Macros.h} (99%) rename cpp/core/utils/{metrics.h => Metrics.h} (100%) rename cpp/core/utils/qpl/{qpl_codec.cc => QplCodec.cc} (99%) rename cpp/core/utils/qpl/{qpl_codec.h => QplCodec.h} (97%) rename cpp/core/utils/qpl/{qpl_job_pool.cc => QplJobPool.cc} (98%) rename cpp/core/utils/qpl/{qpl_job_pool.h => QplJobPool.h} (98%) delete mode 100644 gluten-arrow/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt index 270468275a9b..70ef21d53034 100644 --- a/cpp/core/CMakeLists.txt +++ b/cpp/core/CMakeLists.txt @@ -257,8 +257,7 @@ endif() if(ENABLE_IAA) include(BuildQpl) target_include_directories(gluten PUBLIC ${QPL_INCLUDE_DIR}) - target_sources(gluten PRIVATE utils/qpl/qpl_job_pool.cc - utils/qpl/qpl_codec.cc) + target_sources(gluten PRIVATE utils/qpl/QplJobPool.cc utils/qpl/QplCodec.cc) target_link_libraries(gluten PUBLIC qpl::qpl) endif() diff --git a/cpp/core/benchmarks/CompressionBenchmark.cc b/cpp/core/benchmarks/CompressionBenchmark.cc index 1966c8ef1c18..e1aa69ff39a7 100644 --- a/cpp/core/benchmarks/CompressionBenchmark.cc +++ b/cpp/core/benchmarks/CompressionBenchmark.cc @@ -36,7 +36,7 @@ #include "shuffle/ShuffleWriter.h" #include "utils/Compression.h" -#include "utils/macros.h" +#include "utils/Macros.h" void printTrace(void) { char** strings; diff --git a/cpp/core/compute/ProtobufUtils.cc b/cpp/core/compute/ProtobufUtils.cc index 3bb7d12c7726..c41f062ac5fc 100644 --- a/cpp/core/compute/ProtobufUtils.cc +++ b/cpp/core/compute/ProtobufUtils.cc @@ -22,7 +22,7 @@ #include #include -#include "utils/exception.h" +#include "utils/Exception.h" namespace gluten { diff --git a/cpp/core/compute/ResultIterator.h b/cpp/core/compute/ResultIterator.h index 041f76b22d56..f6d98b9ffafb 100644 --- a/cpp/core/compute/ResultIterator.h +++ b/cpp/core/compute/ResultIterator.h @@ -19,7 +19,7 @@ #include "memory/ColumnarBatch.h" #include "memory/ColumnarBatchIterator.h" -#include "utils/metrics.h" +#include "utils/Metrics.h" namespace gluten { diff --git a/cpp/core/jni/JniCommon.h b/cpp/core/jni/JniCommon.h index 1d784f3a5eda..aeab454f1aab 100644 --- a/cpp/core/jni/JniCommon.h +++ b/cpp/core/jni/JniCommon.h @@ -28,9 +28,9 @@ #include "memory/AllocationListener.h" #include "shuffle/rss/RssClient.h" #include "utils/Compression.h" +#include "utils/Exception.h" #include "utils/ObjectStore.h" #include "utils/ResourceMap.h" -#include "utils/exception.h" static jint jniVersion = JNI_VERSION_1_8; diff --git a/cpp/core/jni/JniError.h b/cpp/core/jni/JniError.h index 828c90861292..1c57dc3b347b 100644 --- a/cpp/core/jni/JniError.h +++ b/cpp/core/jni/JniError.h @@ -20,7 +20,7 @@ #include #include "JniCommon.h" -#include "utils/exception.h" +#include "utils/Exception.h" #ifndef JNI_METHOD_START #define JNI_METHOD_START try { diff --git a/cpp/core/memory/ArrowMemoryPool.cc b/cpp/core/memory/ArrowMemoryPool.cc index 28dd88fc1fab..e1ecae142b6e 100644 --- a/cpp/core/memory/ArrowMemoryPool.cc +++ b/cpp/core/memory/ArrowMemoryPool.cc @@ -17,7 +17,7 @@ #include "ArrowMemoryPool.h" #include "arrow/type_fwd.h" -#include "utils/exception.h" +#include "utils/Exception.h" namespace gluten { diff --git a/cpp/core/memory/ColumnarBatch.cc b/cpp/core/memory/ColumnarBatch.cc index 23567535d50a..66834c7e3971 100644 --- a/cpp/core/memory/ColumnarBatch.cc +++ b/cpp/core/memory/ColumnarBatch.cc @@ -23,7 +23,7 @@ #include "memory/MemoryManager.h" #include "operators/writer/ArrowWriter.h" #include "utils/ArrowStatus.h" -#include "utils/exception.h" +#include "utils/Exception.h" #include "ColumnarBatch.h" diff --git a/cpp/core/memory/ColumnarBatch.h b/cpp/core/memory/ColumnarBatch.h index fd8189aa6a20..2495de248595 100644 --- a/cpp/core/memory/ColumnarBatch.h +++ b/cpp/core/memory/ColumnarBatch.h @@ -25,7 +25,7 @@ #include "memory/MemoryManager.h" #include "operators/writer/ArrowWriter.h" #include "utils/ArrowStatus.h" -#include "utils/exception.h" +#include "utils/Exception.h" namespace gluten { diff --git a/cpp/core/memory/MemoryAllocator.cc b/cpp/core/memory/MemoryAllocator.cc index c637c6a9c13d..277922f70e4b 100644 --- a/cpp/core/memory/MemoryAllocator.cc +++ b/cpp/core/memory/MemoryAllocator.cc @@ -17,7 +17,7 @@ #include "MemoryAllocator.h" #include "HbwAllocator.h" -#include "utils/macros.h" +#include "utils/Macros.h" namespace gluten { diff --git a/cpp/core/operators/r2c/RowToColumnar.h b/cpp/core/operators/r2c/RowToColumnar.h index 32e44c54001e..df4c1151587a 100644 --- a/cpp/core/operators/r2c/RowToColumnar.h +++ b/cpp/core/operators/r2c/RowToColumnar.h @@ -19,7 +19,7 @@ #include #include "memory/ColumnarBatch.h" -#include "utils/exception.h" +#include "utils/Exception.h" namespace gluten { diff --git a/cpp/core/shuffle/LocalPartitionWriter.h b/cpp/core/shuffle/LocalPartitionWriter.h index 826eb22f7431..2b86a81e883b 100644 --- a/cpp/core/shuffle/LocalPartitionWriter.h +++ b/cpp/core/shuffle/LocalPartitionWriter.h @@ -22,7 +22,7 @@ #include "shuffle/PartitionWriter.h" #include "shuffle/ShuffleWriter.h" -#include "utils/macros.h" +#include "utils/Macros.h" namespace gluten { diff --git a/cpp/core/shuffle/Partitioner.cc b/cpp/core/shuffle/Partitioner.cc index 70385a10baad..16d758f6dd75 100644 --- a/cpp/core/shuffle/Partitioner.cc +++ b/cpp/core/shuffle/Partitioner.cc @@ -21,7 +21,7 @@ #include "shuffle/RandomPartitioner.h" #include "shuffle/RoundRobinPartitioner.h" #include "shuffle/SinglePartitioner.h" -#include "utils/exception.h" +#include "utils/Exception.h" namespace gluten { diff --git a/cpp/core/shuffle/Partitioning.cc b/cpp/core/shuffle/Partitioning.cc index 84fe6ecd972f..1a1762cf4266 100644 --- a/cpp/core/shuffle/Partitioning.cc +++ b/cpp/core/shuffle/Partitioning.cc @@ -16,7 +16,7 @@ */ #include "shuffle/Partitioning.h" -#include "utils/exception.h" +#include "utils/Exception.h" namespace { static const std::string kSinglePartitioningName = "single"; diff --git a/cpp/core/shuffle/Payload.cc b/cpp/core/shuffle/Payload.cc index 62e1823fe874..d5cdb6d08eda 100644 --- a/cpp/core/shuffle/Payload.cc +++ b/cpp/core/shuffle/Payload.cc @@ -24,8 +24,8 @@ #include "shuffle/Options.h" #include "shuffle/Utils.h" +#include "utils/Exception.h" #include "utils/Timer.h" -#include "utils/exception.h" namespace gluten { namespace { diff --git a/cpp/core/shuffle/ShuffleReader.cc b/cpp/core/shuffle/ShuffleReader.cc index 5605661df4a4..ced80b3de13f 100644 --- a/cpp/core/shuffle/ShuffleReader.cc +++ b/cpp/core/shuffle/ShuffleReader.cc @@ -18,7 +18,7 @@ #include "ShuffleReader.h" #include "arrow/ipc/reader.h" #include "arrow/record_batch.h" -#include "utils/macros.h" +#include "utils/Macros.h" #include diff --git a/cpp/core/shuffle/ShuffleWriter.cc b/cpp/core/shuffle/ShuffleWriter.cc index 3eff9a2c821e..eff5523fe33c 100644 --- a/cpp/core/shuffle/ShuffleWriter.cc +++ b/cpp/core/shuffle/ShuffleWriter.cc @@ -16,7 +16,7 @@ */ #include "shuffle/ShuffleWriter.h" -#include "utils/exception.h" +#include "utils/Exception.h" namespace gluten { diff --git a/cpp/core/shuffle/Spill.h b/cpp/core/shuffle/Spill.h index 9d8d240879f9..c82a60f562b4 100644 --- a/cpp/core/shuffle/Spill.h +++ b/cpp/core/shuffle/Spill.h @@ -23,7 +23,7 @@ #include #include "shuffle/Payload.h" -#include "utils/macros.h" +#include "utils/Macros.h" namespace gluten { diff --git a/cpp/core/shuffle/rss/RssPartitionWriter.h b/cpp/core/shuffle/rss/RssPartitionWriter.h index dbcbbbb88129..2034c321e163 100644 --- a/cpp/core/shuffle/rss/RssPartitionWriter.h +++ b/cpp/core/shuffle/rss/RssPartitionWriter.h @@ -22,7 +22,7 @@ #include "shuffle/PartitionWriter.h" #include "shuffle/rss/RssClient.h" -#include "utils/macros.h" +#include "utils/Macros.h" namespace gluten { diff --git a/cpp/core/utils/Compression.cc b/cpp/core/utils/Compression.cc index 0d2e03f85643..76e33312677b 100644 --- a/cpp/core/utils/Compression.cc +++ b/cpp/core/utils/Compression.cc @@ -17,14 +17,14 @@ #include "utils/Compression.h" -#include "exception.h" +#include "Exception.h" #ifdef GLUTEN_ENABLE_QAT #include "utils/qat/QatCodec.h" #endif #ifdef GLUTEN_ENABLE_IAA -#include "utils/qpl/qpl_codec.h" +#include "utils/qpl/QplCodec.h" #endif namespace gluten { diff --git a/cpp/core/utils/exception.h b/cpp/core/utils/Exception.h similarity index 100% rename from cpp/core/utils/exception.h rename to cpp/core/utils/Exception.h diff --git a/cpp/core/utils/macros.h b/cpp/core/utils/Macros.h similarity index 99% rename from cpp/core/utils/macros.h rename to cpp/core/utils/Macros.h index fef8ae1e75de..8c6807c40b87 100644 --- a/cpp/core/utils/macros.h +++ b/cpp/core/utils/Macros.h @@ -23,7 +23,7 @@ #include #include -#include "utils/exception.h" +#include "utils/Exception.h" #define GLUTEN_EXPAND(x) x #define GLUTEN_STRINGIFY(x) #x diff --git a/cpp/core/utils/metrics.h b/cpp/core/utils/Metrics.h similarity index 100% rename from cpp/core/utils/metrics.h rename to cpp/core/utils/Metrics.h diff --git a/cpp/core/utils/ResourceMap.h b/cpp/core/utils/ResourceMap.h index b60051afe58e..177981ed4ca0 100644 --- a/cpp/core/utils/ResourceMap.h +++ b/cpp/core/utils/ResourceMap.h @@ -21,7 +21,7 @@ #include #include #include -#include "utils/exception.h" +#include "utils/Exception.h" namespace gluten { using ResourceHandle = uint32_t; diff --git a/cpp/core/utils/StringUtil.cc b/cpp/core/utils/StringUtil.cc index d6acb9e02126..fc6ebb83c0bd 100644 --- a/cpp/core/utils/StringUtil.cc +++ b/cpp/core/utils/StringUtil.cc @@ -20,8 +20,8 @@ #include #include +#include "Exception.h" #include "StringUtil.h" -#include "exception.h" std::vector gluten::splitByDelim(const std::string& s, const char delimiter) { if (s.empty()) { diff --git a/cpp/core/utils/qpl/qpl_codec.cc b/cpp/core/utils/qpl/QplCodec.cc similarity index 99% rename from cpp/core/utils/qpl/qpl_codec.cc rename to cpp/core/utils/qpl/QplCodec.cc index a1d6970b265c..0bf2ec36e72b 100644 --- a/cpp/core/utils/qpl/qpl_codec.cc +++ b/cpp/core/utils/qpl/QplCodec.cc @@ -17,8 +17,8 @@ #include #include -#include -#include +#include +#include #include #include diff --git a/cpp/core/utils/qpl/qpl_codec.h b/cpp/core/utils/qpl/QplCodec.h similarity index 97% rename from cpp/core/utils/qpl/qpl_codec.h rename to cpp/core/utils/qpl/QplCodec.h index 5ddd49bf3256..249b42cd001b 100644 --- a/cpp/core/utils/qpl/qpl_codec.h +++ b/cpp/core/utils/qpl/QplCodec.h @@ -18,7 +18,7 @@ #pragma once #include -#include +#include namespace gluten { namespace qpl { diff --git a/cpp/core/utils/qpl/qpl_job_pool.cc b/cpp/core/utils/qpl/QplJobPool.cc similarity index 98% rename from cpp/core/utils/qpl/qpl_job_pool.cc rename to cpp/core/utils/qpl/QplJobPool.cc index 3d10128a437e..87a4b7687257 100644 --- a/cpp/core/utils/qpl/qpl_job_pool.cc +++ b/cpp/core/utils/qpl/QplJobPool.cc @@ -15,8 +15,8 @@ * limitations under the License. */ -#include "utils/qpl/qpl_job_pool.h" -#include "utils/macros.h" +#include "utils/qpl/QplJobPool.h" +#include "utils/Macros.h" #include #include diff --git a/cpp/core/utils/qpl/qpl_job_pool.h b/cpp/core/utils/qpl/QplJobPool.h similarity index 98% rename from cpp/core/utils/qpl/qpl_job_pool.h rename to cpp/core/utils/qpl/QplJobPool.h index 35ffc262af7e..929f7691faac 100644 --- a/cpp/core/utils/qpl/qpl_job_pool.h +++ b/cpp/core/utils/qpl/QplJobPool.h @@ -26,7 +26,7 @@ #include #include -#include "utils/exception.h" +#include "utils/Exception.h" namespace gluten { namespace qpl { diff --git a/cpp/velox/benchmarks/ColumnarToRowBenchmark.cc b/cpp/velox/benchmarks/ColumnarToRowBenchmark.cc index 8a55050015d0..e2be9eb49621 100644 --- a/cpp/velox/benchmarks/ColumnarToRowBenchmark.cc +++ b/cpp/velox/benchmarks/ColumnarToRowBenchmark.cc @@ -35,9 +35,9 @@ #include "memory/VeloxColumnarBatch.h" #include "memory/VeloxMemoryManager.h" #include "operators/serializer/VeloxColumnarToRowConverter.h" +#include "utils/Macros.h" #include "utils/TestUtils.h" #include "utils/VeloxArrowUtils.h" -#include "utils/macros.h" #include "velox/vector/arrow/Bridge.h" using namespace facebook; diff --git a/cpp/velox/benchmarks/GenericBenchmark.cc b/cpp/velox/benchmarks/GenericBenchmark.cc index 4b46dbd59941..0c520b6bd3e9 100644 --- a/cpp/velox/benchmarks/GenericBenchmark.cc +++ b/cpp/velox/benchmarks/GenericBenchmark.cc @@ -34,10 +34,10 @@ #include "shuffle/LocalPartitionWriter.h" #include "shuffle/VeloxShuffleWriter.h" #include "shuffle/rss/RssPartitionWriter.h" +#include "utils/Exception.h" #include "utils/StringUtil.h" #include "utils/Timer.h" #include "utils/VeloxArrowUtils.h" -#include "utils/exception.h" #include "utils/tests/LocalRssClient.h" #include "velox/exec/PlanNodeStats.h" diff --git a/cpp/velox/benchmarks/ParquetWriteBenchmark.cc b/cpp/velox/benchmarks/ParquetWriteBenchmark.cc index 45348ed4a63b..b1118b6ae0bd 100644 --- a/cpp/velox/benchmarks/ParquetWriteBenchmark.cc +++ b/cpp/velox/benchmarks/ParquetWriteBenchmark.cc @@ -38,9 +38,9 @@ #include "memory/ArrowMemoryPool.h" #include "memory/ColumnarBatch.h" #include "memory/VeloxMemoryManager.h" +#include "utils/Macros.h" #include "utils/TestUtils.h" #include "utils/VeloxArrowUtils.h" -#include "utils/macros.h" #include "velox/dwio/parquet/writer/Writer.h" #include "velox/vector/arrow/Bridge.h" diff --git a/cpp/velox/benchmarks/common/BenchmarkUtils.h b/cpp/velox/benchmarks/common/BenchmarkUtils.h index 181e56807bcd..4f4f5b75df85 100644 --- a/cpp/velox/benchmarks/common/BenchmarkUtils.h +++ b/cpp/velox/benchmarks/common/BenchmarkUtils.h @@ -33,8 +33,8 @@ #include "memory/VeloxMemoryManager.h" #include "shuffle/Options.h" #include "shuffle/ShuffleWriter.h" +#include "utils/Exception.h" #include "utils/VeloxArrowUtils.h" -#include "utils/exception.h" #include "velox/common/memory/Memory.h" DECLARE_int64(batch_size); diff --git a/cpp/velox/benchmarks/common/ParquetReaderIterator.h b/cpp/velox/benchmarks/common/ParquetReaderIterator.h index 6d162e4b68d5..20652ee27dc0 100644 --- a/cpp/velox/benchmarks/common/ParquetReaderIterator.h +++ b/cpp/velox/benchmarks/common/ParquetReaderIterator.h @@ -17,7 +17,7 @@ #include "benchmarks/common/BenchmarkUtils.h" #include "benchmarks/common/FileReaderIterator.h" -#include "utils/macros.h" +#include "utils/Macros.h" #include diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index ebe4426db4f3..5110590d0277 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -28,13 +28,13 @@ #include "utils/qat/QatCodec.h" #endif #ifdef GLUTEN_ENABLE_IAA -#include "utils/qpl/qpl_codec.h" +#include "utils/qpl/QplCodec.h" #endif #include "compute/VeloxRuntime.h" #include "config/VeloxConfig.h" #include "jni/JniFileSystem.h" #include "udf/UdfLoader.h" -#include "utils/exception.h" +#include "utils/Exception.h" #include "velox/common/caching/SsdCache.h" #include "velox/common/file/FileSystems.h" #include "velox/connectors/hive/HiveConnector.h" diff --git a/cpp/velox/compute/WholeStageResultIterator.h b/cpp/velox/compute/WholeStageResultIterator.h index 50ec1201e542..9b1f2b728802 100644 --- a/cpp/velox/compute/WholeStageResultIterator.h +++ b/cpp/velox/compute/WholeStageResultIterator.h @@ -22,7 +22,7 @@ #include "memory/VeloxColumnarBatch.h" #include "substrait/SubstraitToVeloxPlan.h" #include "substrait/plan.pb.h" -#include "utils/metrics.h" +#include "utils/Metrics.h" #include "velox/common/config/Config.h" #include "velox/connectors/hive/iceberg/IcebergSplit.h" #include "velox/core/PlanNode.h" diff --git a/cpp/velox/jni/JniUdf.cc b/cpp/velox/jni/JniUdf.cc index aa5d1c0932c4..32514e7ae79c 100644 --- a/cpp/velox/jni/JniUdf.cc +++ b/cpp/velox/jni/JniUdf.cc @@ -18,7 +18,7 @@ #include "JniUdf.h" #include "jni/JniCommon.h" #include "udf/UdfLoader.h" -#include "utils/exception.h" +#include "utils/Exception.h" namespace { diff --git a/cpp/velox/memory/ArrowMemory.h b/cpp/velox/memory/ArrowMemory.h index 6bba740b8bf1..0be5819ef796 100644 --- a/cpp/velox/memory/ArrowMemory.h +++ b/cpp/velox/memory/ArrowMemory.h @@ -18,7 +18,7 @@ #pragma once #include -#include "utils/exception.h" +#include "utils/Exception.h" #include "velox/common/memory/ByteStream.h" namespace gluten { diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc index cc1419e85b05..c3845d1f0df0 100644 --- a/cpp/velox/memory/VeloxMemoryManager.cc +++ b/cpp/velox/memory/VeloxMemoryManager.cc @@ -27,7 +27,7 @@ #include "compute/VeloxBackend.h" #include "config/VeloxConfig.h" #include "memory/ArrowMemoryPool.h" -#include "utils/exception.h" +#include "utils/Exception.h" DECLARE_int32(gluten_velox_aysnc_timeout_on_task_stopping); diff --git a/cpp/velox/operators/serializer/VeloxColumnarToRowConverter.cc b/cpp/velox/operators/serializer/VeloxColumnarToRowConverter.cc index a609e26d0390..f9c910a6244a 100644 --- a/cpp/velox/operators/serializer/VeloxColumnarToRowConverter.cc +++ b/cpp/velox/operators/serializer/VeloxColumnarToRowConverter.cc @@ -20,7 +20,7 @@ #include #include "memory/VeloxColumnarBatch.h" -#include "utils/exception.h" +#include "utils/Exception.h" #include "velox/row/UnsafeRowDeserializers.h" #include "velox/row/UnsafeRowFast.h" diff --git a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc index 49ed14245f94..4001a91decf4 100644 --- a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc +++ b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc @@ -22,8 +22,8 @@ #include "shuffle/ShuffleSchema.h" #include "shuffle/Utils.h" #include "utils/Common.h" +#include "utils/Macros.h" #include "utils/VeloxArrowUtils.h" -#include "utils/macros.h" #include "velox/buffer/Buffer.h" #include "velox/common/base/Nulls.h" #include "velox/type/HugeInt.h" diff --git a/cpp/velox/shuffle/VeloxRssSortShuffleWriter.cc b/cpp/velox/shuffle/VeloxRssSortShuffleWriter.cc index b50bf0fad8a6..a3fdf5e273c7 100644 --- a/cpp/velox/shuffle/VeloxRssSortShuffleWriter.cc +++ b/cpp/velox/shuffle/VeloxRssSortShuffleWriter.cc @@ -20,8 +20,8 @@ #include "memory/VeloxMemoryManager.h" #include "shuffle/ShuffleSchema.h" #include "utils/Common.h" +#include "utils/Macros.h" #include "utils/VeloxArrowUtils.h" -#include "utils/macros.h" #include "velox/common/base/Nulls.h" #include "velox/type/Type.h" diff --git a/cpp/velox/shuffle/VeloxShuffleReader.cc b/cpp/velox/shuffle/VeloxShuffleReader.cc index 4eec461e6532..03ffce2db01a 100644 --- a/cpp/velox/shuffle/VeloxShuffleReader.cc +++ b/cpp/velox/shuffle/VeloxShuffleReader.cc @@ -26,9 +26,9 @@ #include "shuffle/Utils.h" #include "utils/Common.h" #include "utils/Compression.h" +#include "utils/Macros.h" #include "utils/Timer.h" #include "utils/VeloxArrowUtils.h" -#include "utils/macros.h" #include "velox/row/CompactRow.h" #include "velox/serializers/PrestoSerializer.h" #include "velox/vector/ComplexVector.h" diff --git a/cpp/velox/substrait/VeloxToSubstraitPlan.cc b/cpp/velox/substrait/VeloxToSubstraitPlan.cc index 4908f4dc8f88..19259f81f629 100644 --- a/cpp/velox/substrait/VeloxToSubstraitPlan.cc +++ b/cpp/velox/substrait/VeloxToSubstraitPlan.cc @@ -17,7 +17,7 @@ #include "VeloxToSubstraitPlan.h" #include -#include "utils/exception.h" +#include "utils/Exception.h" namespace gluten { diff --git a/cpp/velox/udf/UdfLoader.cc b/cpp/velox/udf/UdfLoader.cc index 8a9918166218..dfb10275dfea 100644 --- a/cpp/velox/udf/UdfLoader.cc +++ b/cpp/velox/udf/UdfLoader.cc @@ -25,9 +25,9 @@ #include "Udaf.h" #include "Udf.h" #include "UdfLoader.h" +#include "utils/Exception.h" +#include "utils/Macros.h" #include "utils/StringUtil.h" -#include "utils/exception.h" -#include "utils/macros.h" namespace { diff --git a/cpp/velox/utils/ConfigExtractor.cc b/cpp/velox/utils/ConfigExtractor.cc index 4189df758f39..b07c57c019f5 100644 --- a/cpp/velox/utils/ConfigExtractor.cc +++ b/cpp/velox/utils/ConfigExtractor.cc @@ -23,7 +23,7 @@ #include #endif -#include "utils/exception.h" +#include "utils/Exception.h" #include "velox/connectors/hive/HiveConfig.h" namespace { diff --git a/cpp/velox/utils/HdfsUtils.cc b/cpp/velox/utils/HdfsUtils.cc index 1bc326d4ca48..ec395f817cdf 100644 --- a/cpp/velox/utils/HdfsUtils.cc +++ b/cpp/velox/utils/HdfsUtils.cc @@ -18,7 +18,7 @@ #include "HdfsUtils.h" #include #include "config/GlutenConfig.h" -#include "utils/exception.h" +#include "utils/Exception.h" namespace gluten { diff --git a/cpp/velox/utils/VeloxBatchResizer.h b/cpp/velox/utils/VeloxBatchResizer.h index 09cdf4d51170..522baf4982e4 100644 --- a/cpp/velox/utils/VeloxBatchResizer.h +++ b/cpp/velox/utils/VeloxBatchResizer.h @@ -17,7 +17,7 @@ #include "memory/ColumnarBatchIterator.h" #include "memory/VeloxColumnarBatch.h" -#include "utils/exception.h" +#include "utils/Exception.h" #include "velox/common/memory/MemoryPool.h" #include "velox/vector/ComplexVector.h" diff --git a/cpp/velox/utils/tests/LocalRssClient.h b/cpp/velox/utils/tests/LocalRssClient.h index c5c1b5d2c359..ff2f89ef7fec 100644 --- a/cpp/velox/utils/tests/LocalRssClient.h +++ b/cpp/velox/utils/tests/LocalRssClient.h @@ -22,7 +22,7 @@ #include #include "shuffle/rss/RssClient.h" #include "utils/Common.h" -#include "utils/macros.h" +#include "utils/Macros.h" namespace gluten { diff --git a/cpp/velox/utils/tests/MemoryPoolUtils.h b/cpp/velox/utils/tests/MemoryPoolUtils.h index 01d4f2d0d33c..68e279008497 100644 --- a/cpp/velox/utils/tests/MemoryPoolUtils.h +++ b/cpp/velox/utils/tests/MemoryPoolUtils.h @@ -19,7 +19,7 @@ #include #include "memory/Reclaimable.h" -#include "utils/exception.h" +#include "utils/Exception.h" #include "velox/common/base/Exceptions.h" namespace gluten { diff --git a/gluten-arrow/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java b/gluten-arrow/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java index fd9c72c36060..e6d97db84b12 100644 --- a/gluten-arrow/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java +++ b/gluten-arrow/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java @@ -21,7 +21,6 @@ import org.apache.gluten.runtime.Runtimes; import org.apache.gluten.utils.ArrowAbiUtil; import org.apache.gluten.utils.ArrowUtil; -import org.apache.gluten.utils.ImplicitClass; import org.apache.gluten.utils.InternalRowUtl; import org.apache.gluten.vectorized.ArrowWritableColumnVector; @@ -168,10 +167,11 @@ private static ColumnarBatch load(BufferAllocator allocator, ColumnarBatch input // Follow gluten input's reference count. This might be optimized using // automatic clean-up or once the extensibility of ColumnarBatch is enriched IndicatorVector giv = (IndicatorVector) input.column(0); - ImplicitClass.ArrowColumnarBatchRetainer retainer = - new ImplicitClass.ArrowColumnarBatchRetainer(output); for (long i = 0; i < (giv.refCnt() - 1); i++) { - retainer.retain(); + for (int j = 0; j < output.numCols(); j++) { + final ArrowWritableColumnVector col = (ArrowWritableColumnVector) output.column(j); + col.retain(); + } } // close the input one diff --git a/gluten-arrow/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala b/gluten-arrow/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala deleted file mode 100644 index 4ffb3ab5cdcc..000000000000 --- a/gluten-arrow/src/main/scala/org/apache/gluten/utils/ImplicitClass.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.utils - -import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators -import org.apache.gluten.vectorized.ArrowWritableColumnVector - -import org.apache.spark.sql.vectorized.ColumnarBatch - -object ImplicitClass { - - implicit class ArrowColumnarBatchRetainer(val cb: ColumnarBatch) { - def retain(): Unit = { - (0 until cb.numCols).toList.foreach( - i => - ColumnarBatches - .ensureLoaded(ArrowBufferAllocators.contextInstance(), cb) - .column(i) - .asInstanceOf[ArrowWritableColumnVector] - .retain()) - } - } -}