Skip to content

Commit

Permalink
Organize functions in ReaderTest (facebookincubator#7741)
Browse files Browse the repository at this point in the history
Summary:
Pull Request resolved: facebookincubator#7741

Putting `verifyFlatMapReading` overloads next to each other

Reviewed By: DanielMunozT

Differential Revision: D51352170

fbshipit-source-id: 1a89a9c4b1aa9f7930bfe393dc93105a3017e2d6
  • Loading branch information
Patrick Sullivan authored and facebook-github-bot committed Dec 8, 2023
1 parent d52d68b commit 6667e45
Showing 1 changed file with 36 additions and 34 deletions.
70 changes: 36 additions & 34 deletions velox/dwio/dwrf/test/ReaderTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,42 @@ std::unique_ptr<BufferedInput> createFileBufferedInput(
std::make_shared<LocalReadFile>(path), pool);
}

// Prefetches the entire range of the reader and verifies correctness in
// PrefetchUnits() API. Does not do any actual reading of the file.
void verifyPrefetch(
DwrfRowReader* rowReader,
const std::vector<uint32_t>& expectedPrefetchRowSizes = {},
const std::vector<bool>& shouldTryPrefetch = {}) {
auto prefetchUnitsOpt = rowReader->prefetchUnits();
ASSERT_TRUE(prefetchUnitsOpt.has_value());
auto prefetchUnits = std::move(prefetchUnitsOpt.value());
auto numFetches = prefetchUnits.size();
auto expectedResultsSize = shouldTryPrefetch.size();
auto expectedRowsSize = expectedPrefetchRowSizes.size();
bool shouldCheckResults = expectedResultsSize != 0;
bool shouldCheckRowCount = expectedRowsSize != 0;

// Empty vector will skip the check, but they should never been different than
// actual expected prefetchUnits vector
DWIO_ENSURE(expectedResultsSize == numFetches || !shouldCheckResults);
DWIO_ENSURE(expectedRowsSize == numFetches || !shouldCheckRowCount);

for (int i = 0; i < numFetches; i++) {
if (shouldCheckRowCount) {
EXPECT_EQ(prefetchUnits[i].rowCount, expectedPrefetchRowSizes[i]);
}
if (shouldCheckResults && shouldTryPrefetch[i]) {
RowReader::FetchResult result = prefetchUnits[i].prefetch();
EXPECT_EQ(
result,
// A prefetch request for the first stripe should be already fetched,
// because createDwrfRowReader calls startNextStripe() synchronously.
i == 0 ? RowReader::FetchResult::kAlreadyFetched
: RowReader::FetchResult::kFetched);
}
}
}

// This relies on schema and data inside of our fm_small and fm_large orc files,
// and is not composeable with other schema/datas
void verifyFlatMapReading(
Expand Down Expand Up @@ -222,40 +258,6 @@ void verifyFlatMapReading(
EXPECT_EQ(batchId, numBatches);
}

void verifyPrefetch(
DwrfRowReader* rowReader,
const std::vector<uint32_t>& expectedPrefetchRowSizes = {},
const std::vector<bool>& shouldTryPrefetch = {}) {
auto prefetchUnitsOpt = rowReader->prefetchUnits();
ASSERT_TRUE(prefetchUnitsOpt.has_value());
auto prefetchUnits = std::move(prefetchUnitsOpt.value());
auto numFetches = prefetchUnits.size();
auto expectedResultsSize = shouldTryPrefetch.size();
auto expectedRowsSize = expectedPrefetchRowSizes.size();
bool shouldCheckResults = expectedResultsSize != 0;
bool shouldCheckRowCount = expectedRowsSize != 0;

// Empty vector will skip the check, but they should never been different than
// actual expected prefetchUnits vector
DWIO_ENSURE(expectedResultsSize == numFetches || !shouldCheckResults);
DWIO_ENSURE(expectedRowsSize == numFetches || !shouldCheckRowCount);

for (int i = 0; i < numFetches; i++) {
if (shouldCheckRowCount) {
EXPECT_EQ(prefetchUnits[i].rowCount, expectedPrefetchRowSizes[i]);
}
if (shouldCheckResults && shouldTryPrefetch[i]) {
RowReader::FetchResult result = prefetchUnits[i].prefetch();
EXPECT_EQ(
result,
// A prefetch request for the first stripe should be already fetched,
// because createDwrfRowReader calls startNextStripe() synchronously.
i == 0 ? RowReader::FetchResult::kAlreadyFetched
: RowReader::FetchResult::kFetched);
}
}
}

// schema of flat map sample file
// struct {
// id int,
Expand Down

0 comments on commit 6667e45

Please sign in to comment.