Skip to content

Commit

Permalink
fix RecordBatch size in hash join (#13916)
Browse files Browse the repository at this point in the history
  • Loading branch information
getChan authored Dec 29, 2024
1 parent a203c2b commit a47729c
Showing 1 changed file with 12 additions and 1 deletion.
13 changes: 12 additions & 1 deletion datafusion/physical-plan/src/joins/hash_join.rs
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ use datafusion_physical_expr::equivalence::{
};
use datafusion_physical_expr::PhysicalExprRef;

use crate::spill::get_record_batch_memory_size;
use ahash::RandomState;
use datafusion_expr::Operator;
use datafusion_physical_expr_common::datum::compare_op_for_nested;
Expand Down Expand Up @@ -921,7 +922,7 @@ async fn collect_left_input(
let initial = (Vec::new(), 0, metrics, reservation);
let (batches, num_rows, metrics, mut reservation) = stream
.try_fold(initial, |mut acc, batch| async {
let batch_size = batch.get_array_memory_size();
let batch_size = get_record_batch_memory_size(&batch);
// Reserve memory for incoming batch
acc.3.try_grow(batch_size)?;
// Update metrics
Expand Down Expand Up @@ -3982,6 +3983,11 @@ mod tests {
err.to_string(),
"External error: Resources exhausted: Additional allocation failed with top memory consumers (across reservations) as: HashJoinInput"
);

assert_contains!(
err.to_string(),
"Failed to allocate additional 120 bytes for HashJoinInput"
);
}

Ok(())
Expand Down Expand Up @@ -4063,6 +4069,11 @@ mod tests {
"External error: Resources exhausted: Additional allocation failed with top memory consumers (across reservations) as: HashJoinInput[1]"

);

assert_contains!(
err.to_string(),
"Failed to allocate additional 120 bytes for HashJoinInput[1]"
);
}

Ok(())
Expand Down

0 comments on commit a47729c

Please sign in to comment.