fix: skip null data type fields (parseablehq#1016)

skip the schema generation with `Null` data type fields fixes the issue when first event has null data type and field has valid data after the initial schema is persisted fixes the schema merge issue for such cases --------- Signed-off-by: Nitish Tiwari <[email protected]> Co-authored-by: Nitish Tiwari <[email protected]> Co-authored-by: Devdutt Shenoi <[email protected]>
parmesant · Dec 4, 2024 · dd2b8b2 · dd2b8b2
1 parent 0353dc4
commit dd2b8b2
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/src/event/format/json.rs b/src/event/format/json.rs
@@ -86,6 +86,7 @@ impl EventFormat for Event {
                     infer_schema
                         .fields
                         .iter()
+                        .filter(|field| !field.data_type().is_null())
                         .cloned()
                         .sorted_by(|a, b| a.name().cmp(b.name()))
                         .collect()

diff --git a/src/handlers/http/ingest.rs b/src/handlers/http/ingest.rs
@@ -384,7 +384,7 @@ mod tests {
         let (rb, _) = into_event_batch(req, json, HashMap::default(), None, None).unwrap();
 
         assert_eq!(rb.num_rows(), 1);
-        assert_eq!(rb.num_columns(), 6);
+        assert_eq!(rb.num_columns(), 5);
         assert_eq!(
             rb.column_by_name("a").unwrap().as_int64_arr(),
             &Int64Array::from_iter([1])