[WIP][VL] Support celeborn sort based shuffle

apache · May 14, 2024 · 56a447f · 56a447f
1 parent 81987a7
commit 56a447f
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 6 deletions.
diff --git a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.cc
@@ -123,8 +123,8 @@ arrow::Result<std::shared_ptr<VeloxSortBasedShuffleWriter>> VeloxSortBasedShuffl
     ShuffleWriterOptions options,
     std::shared_ptr<facebook::velox::memory::MemoryPool> veloxPool,
     arrow::MemoryPool* arrowPool) {
-  std::shared_ptr<VeloxSortBasedShuffleWriter> res(
-      new VeloxSortBasedShuffleWriter(numPartitions, std::move(partitionWriter), std::move(options), veloxPool, arrowPool));
+  std::shared_ptr<VeloxSortBasedShuffleWriter> res(new VeloxSortBasedShuffleWriter(
+      numPartitions, std::move(partitionWriter), std::move(options), veloxPool, arrowPool));
   RETURN_NOT_OK(res->init());
   return res;
 } // namespace gluten
@@ -307,9 +307,7 @@ arrow::Status VeloxSortBasedShuffleWriter::initFromRowVector(const facebook::vel
       false, facebook::velox::common::stringToCompressionKind(partitionWriter_->options().compressionTypeStr)};
   batch_ = std::make_unique<facebook::velox::VectorStreamGroup>(veloxPool_.get());
   batch_->createStreamTree(
-      std::static_pointer_cast<const facebook::velox::RowType>(rowType_.value()),
-      options_.bufferSize,
-      &serdeOptions_);
+      std::static_pointer_cast<const facebook::velox::RowType>(rowType_.value()), options_.bufferSize, &serdeOptions_);
   return arrow::Status::OK();
 }
 

diff --git a/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h b/cpp/velox/shuffle/VeloxSortBasedShuffleWriter.h
@@ -101,7 +101,6 @@ struct BinaryArrayResizeState {
 };
 
 class VeloxSortBasedShuffleWriter : public ShuffleWriter {
-
  public:
   static arrow::Result<std::shared_ptr<VeloxSortBasedShuffleWriter>> create(
       uint32_t numPartitions,
-Original file line number
+Diff line change
@@ Expand Up / @@ -101,7 +101,6 @@ struct BinaryArrayResizeState { @@
     };
     class VeloxSortBasedShuffleWriter : public ShuffleWriter {
      public:
       static arrow::Result<std::shared_ptr<VeloxSortBasedShuffleWriter>> create(
           uint32_t numPartitions,
@@ Expand Down @@