From 35b096fe1f1431ca5a57b6d908622b0a3edd6ed1 Mon Sep 17 00:00:00 2001 From: Jialiang Tan Date: Fri, 25 Oct 2024 16:17:36 -0700 Subject: [PATCH] QueryConfig clean up --- velox/core/QueryConfig.h | 49 +++++++++++++--------------------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/velox/core/QueryConfig.h b/velox/core/QueryConfig.h index 3910b58794666..354f0ff9b476e 100644 --- a/velox/core/QueryConfig.h +++ b/velox/core/QueryConfig.h @@ -146,9 +146,20 @@ class QueryConfig { static constexpr const char* kAbandonPartialTopNRowNumberMinPct = "abandon_partial_topn_row_number_min_pct"; + /// The maximum number of bytes to buffer in PartitionedOutput operator to + /// avoid creating tiny SerializedPages. + /// + /// For PartitionedOutputNode::Kind::kPartitioned, PartitionedOutput operator + /// would buffer up to that number of bytes / number of destinations for each + /// destination before producing a SerializedPage. static constexpr const char* kMaxPartitionedOutputBufferSize = "max_page_partitioning_buffer_size"; + /// The maximum size in bytes for the task's buffered output. + /// + /// The producer Drivers are blocked when the buffered size exceeds + /// this. The Drivers are resumed when the buffered size goes below + /// OutputBufferManager::kContinuePct % of this. static constexpr const char* kMaxOutputBufferSize = "max_output_buffer_size"; /// Preferred size of batches in bytes to be returned by operators from @@ -268,10 +279,6 @@ class QueryConfig { static constexpr const char* kSpillNumPartitionBits = "spiller_num_partition_bits"; - /// !!! DEPRECATED: do not use. - static constexpr const char* kJoinSpillPartitionBits = - "join_spiller_partition_bits"; - static constexpr const char* kMinSpillableReservationPct = "min_spillable_reservation_pct"; @@ -287,7 +294,7 @@ class QueryConfig { static constexpr const char* kPrestoArrayAggIgnoreNulls = "presto.array_agg.ignore_nulls"; - // The default number of expected items for the bloomfilter. + /// The default number of expected items for the bloomfilter. static constexpr const char* kSparkBloomFilterExpectedNumItems = "spark.bloom_filter.expected_num_items"; @@ -341,10 +348,10 @@ class QueryConfig { static constexpr const char* kEnableExpressionEvaluationCache = "enable_expression_evaluation_cache"; - // For a given shared subexpression, the maximum distinct sets of inputs we - // cache results for. Lambdas can call the same expression with different - // inputs many times, causing the results we cache to explode in size. Putting - // a limit contains the memory usage. + /// For a given shared subexpression, the maximum distinct sets of inputs we + /// cache results for. Lambdas can call the same expression with different + /// inputs many times, causing the results we cache to explode in size. + /// Putting a limit contains the memory usage. static constexpr const char* kMaxSharedSubexprResultsCached = "max_shared_subexpr_results_cached"; @@ -474,22 +481,11 @@ class QueryConfig { return get(kMaxSpillBytes, kDefault); } - /// Returns the maximum number of bytes to buffer in PartitionedOutput - /// operator to avoid creating tiny SerializedPages. - /// - /// For PartitionedOutputNode::Kind::kPartitioned, PartitionedOutput operator - /// would buffer up to that number of bytes / number of destinations for each - /// destination before producing a SerializedPage. uint64_t maxPartitionedOutputBufferSize() const { static constexpr uint64_t kDefault = 32UL << 20; return get(kMaxPartitionedOutputBufferSize, kDefault); } - /// Returns the maximum size in bytes for the task's buffered output. - /// - /// The producer Drivers are blocked when the buffered size exceeds - /// this. The Drivers are resumed when the buffered size goes below - /// OutputBufferManager::kContinuePct % of this. uint64_t maxOutputBufferSize() const { static constexpr uint64_t kDefault = 32UL << 20; return get(kMaxOutputBufferSize, kDefault); @@ -640,19 +636,6 @@ class QueryConfig { return get(kSpillStartPartitionBit, kDefaultStartBit); } - /// Returns the number of bits used to calculate the spill partition number - /// for hash join. The number of spill partitions will be power of two. - /// - /// NOTE: as for now, we only support up to 8-way spill partitioning. - /// - /// DEPRECATED. - uint8_t joinSpillPartitionBits() const { - constexpr uint8_t kDefaultBits = 3; - constexpr uint8_t kMaxBits = 3; - return std::min( - kMaxBits, get(kJoinSpillPartitionBits, kDefaultBits)); - } - /// Returns the number of bits used to calculate the spill partition number /// for hash join and RowNumber. The number of spill partitions will be power /// of two.