From 89b902c207738bd06eedfac264c0b19624c4e816 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Thu, 5 Sep 2024 01:53:50 +0000 Subject: [PATCH] address comments --- cpp/velox/shuffle/VeloxShuffleReader.cc | 3 ++- cpp/velox/shuffle/VeloxSortShuffleWriter.cc | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/velox/shuffle/VeloxShuffleReader.cc b/cpp/velox/shuffle/VeloxShuffleReader.cc index d5fe07f9008b..173e72a2073f 100644 --- a/cpp/velox/shuffle/VeloxShuffleReader.cc +++ b/cpp/velox/shuffle/VeloxShuffleReader.cc @@ -416,11 +416,12 @@ std::shared_ptr VeloxSortShuffleReaderDeserializer::next() { cachedInputs_.emplace_back(numRows, wrapInBufferViewAsOwner(buffer->data(), buffer->size(), buffer)); cachedRows_ += numRows; } else { - // For a large row, read all segments. + // numRows = 0 indicates a segment of a large row. std::vector> buffers; auto rowSize = *reinterpret_cast(const_cast(arrowBuffers[0]->data())); RowSizeType bufferSize = arrowBuffers[0]->size(); buffers.emplace_back(std::move(arrowBuffers[0])); + // Read remaining segments. while (bufferSize < rowSize) { GLUTEN_ASSIGN_OR_THROW( arrowBuffers, BlockPayload::deserialize(in_.get(), codec_, arrowPool_, numRows, decompressTime_)); diff --git a/cpp/velox/shuffle/VeloxSortShuffleWriter.cc b/cpp/velox/shuffle/VeloxSortShuffleWriter.cc index 945e157f2238..55aa739e7a0d 100644 --- a/cpp/velox/shuffle/VeloxSortShuffleWriter.cc +++ b/cpp/velox/shuffle/VeloxSortShuffleWriter.cc @@ -273,7 +273,7 @@ arrow::Status VeloxSortShuffleWriter::evictAllPartitions() { } arrow::Status VeloxSortShuffleWriter::evictPartition(uint32_t partitionId, size_t begin, size_t end) { - VELOX_CHECK(begin < end); + VELOX_DCHECK(begin < end); // Count copy row time into sortTime_. Timer sortTime{}; // Serialize [begin, end)