Skip to content

Commit

Permalink
comparison with structtype schema
Browse files Browse the repository at this point in the history
Signed-off-by: JustinRush80 <[email protected]>
  • Loading branch information
JustinRush80 committed Jan 23, 2025
1 parent 3beae1f commit 0a6e4c6
Showing 1 changed file with 22 additions and 5 deletions.
27 changes: 22 additions & 5 deletions crates/core/src/operations/merge/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ use datafusion_expr::{
UNNAMED_TABLE,
};

use delta_kernel::schema::StructType;
use filter::try_construct_early_filter;
use futures::future::BoxFuture;
use parquet::file::properties::WriterProperties;
Expand Down Expand Up @@ -1001,9 +1002,10 @@ async fn execute(
)?;
let schema = Arc::new(schema_bulider.finish());
new_schema = Some(schema.clone());
if schema != snapshot.input_schema()? {
let schema_struct: StructType = schema.try_into()?;
if &schema_struct != snapshot.schema() {
let schema_action = Action::Metadata(Metadata::try_new(
schema.try_into()?,
schema_struct,
current_metadata.partition_columns.clone(),
snapshot.metadata().configuration.clone(),
)?);
Expand Down Expand Up @@ -1594,6 +1596,7 @@ impl std::future::IntoFuture for MergeBuilder {

#[cfg(test)]
mod tests {
use crate::kernel::Action;
use crate::kernel::DataType;
use crate::kernel::PrimitiveType;
use crate::kernel::StructField;
Expand Down Expand Up @@ -1769,7 +1772,7 @@ mod tests {
.unwrap();

let commit_info = table.history(None).await.unwrap();
dbg!(&commit_info);

let last_commit = &commit_info[0];
let parameters = last_commit.operation_parameters.clone().unwrap();
assert!(!parameters.contains_key("predicate"));
Expand Down Expand Up @@ -1843,7 +1846,7 @@ mod tests {

let commit_info = after_table.history(None).await.unwrap();
let last_commit = &commit_info[0];
dbg!(&commit_info);

let parameters = last_commit.operation_parameters.clone().unwrap();
assert!(!parameters.contains_key("predicate"));
assert_eq!(parameters["mergePredicate"], json!("target.id = source.id"));
Expand All @@ -1861,8 +1864,22 @@ mod tests {
);

assert_eq!(table.schema(), after_table.schema());
dbg!(after_table.snapshot().unwrap().add_actions_table(true));

let snapshot_bytes = after_table
.log_store
.read_commit_entry(2)
.await
.unwrap()
.expect("failed to get snapshot bytes");
let actions = crate::logstore::get_actions(2, snapshot_bytes)
.await
.unwrap();

let schema_actions = actions
.iter()
.any(|action| matches!(action, Action::Metadata(_)));

assert!(!schema_actions);
assert_merge(after_table, metrics).await;
}

Expand Down

0 comments on commit 0a6e4c6

Please sign in to comment.