From dd2b8b260c02c11fe998ccbef967b63b0228723c Mon Sep 17 00:00:00 2001 From: Nikhil Sinha <131262146+nikhilsinhaparseable@users.noreply.github.com> Date: Wed, 4 Dec 2024 11:05:43 -0800 Subject: [PATCH] fix: skip null data type fields (#1016) skip the schema generation with `Null` data type fields fixes the issue when first event has null data type and field has valid data after the initial schema is persisted fixes the schema merge issue for such cases --------- Signed-off-by: Nitish Tiwari Co-authored-by: Nitish Tiwari Co-authored-by: Devdutt Shenoi --- src/event/format/json.rs | 1 + src/handlers/http/ingest.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/event/format/json.rs b/src/event/format/json.rs index cedbabaa..abb823c4 100644 --- a/src/event/format/json.rs +++ b/src/event/format/json.rs @@ -86,6 +86,7 @@ impl EventFormat for Event { infer_schema .fields .iter() + .filter(|field| !field.data_type().is_null()) .cloned() .sorted_by(|a, b| a.name().cmp(b.name())) .collect() diff --git a/src/handlers/http/ingest.rs b/src/handlers/http/ingest.rs index d15f6660..9f72000b 100644 --- a/src/handlers/http/ingest.rs +++ b/src/handlers/http/ingest.rs @@ -384,7 +384,7 @@ mod tests { let (rb, _) = into_event_batch(req, json, HashMap::default(), None, None).unwrap(); assert_eq!(rb.num_rows(), 1); - assert_eq!(rb.num_columns(), 6); + assert_eq!(rb.num_columns(), 5); assert_eq!( rb.column_by_name("a").unwrap().as_int64_arr(), &Int64Array::from_iter([1])