Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update to Arrow 35.0.0 #5441

Closed
wants to merge 8 commits into from
Closed

Update to Arrow 35.0.0 #5441

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
37a197c
Update to NullBuffer in ArrayData
tustvold Mar 1, 2023
4356cd1
Fix test
tustvold Mar 1, 2023
74e2222
Merge remote-tracking branch 'upstream/main' into null-buffer-changes
tustvold Mar 10, 2023
58ad1e7
Update pin
tustvold Mar 10, 2023
a165110
Remove pin
tustvold Mar 14, 2023
08321a5
Update regexpmatch
tustvold Mar 14, 2023
5b83dcf
Merge remote-tracking branch 'upstream/main' into null-buffer-changes
tustvold Mar 14, 2023
06deff0
Fixes
tustvold Mar 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,10 @@ opt-level = 3
overflow-checks = false
panic = 'unwind'
rpath = false

[patch.crates-io]
arrow = { git = "https://github.com/tustvold/arrow-rs.git", rev = "5f22cd4e6fa16729e946cbdfbacfa27692f35ab8" }
arrow-flight = { git = "https://github.com/tustvold/arrow-rs.git", rev = "5f22cd4e6fa16729e946cbdfbacfa27692f35ab8" }
arrow-schema = { git = "https://github.com/tustvold/arrow-rs.git", rev = "5f22cd4e6fa16729e946cbdfbacfa27692f35ab8" }
arrow-buffer = { git = "https://github.com/tustvold/arrow-rs.git", rev = "5f22cd4e6fa16729e946cbdfbacfa27692f35ab8" }
parquet = { git = "https://github.com/tustvold/arrow-rs.git", rev = "5f22cd4e6fa16729e946cbdfbacfa27692f35ab8" }
45 changes: 15 additions & 30 deletions datafusion-cli/Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions datafusion-cli/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,9 @@ parking_lot = { version = "0.12" }
rustyline = "10.0"
tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
url = "2.2"

[patch.crates-io]
arrow = { git = "https://github.com/tustvold/arrow-rs.git", rev = "5f22cd4e6fa16729e946cbdfbacfa27692f35ab8" }
arrow-buffer = { git = "https://github.com/tustvold/arrow-rs.git", rev = "5f22cd4e6fa16729e946cbdfbacfa27692f35ab8" }
arrow-schema = { git = "https://github.com/tustvold/arrow-rs.git", rev = "5f22cd4e6fa16729e946cbdfbacfa27692f35ab8" }
parquet = { git = "https://github.com/tustvold/arrow-rs.git", rev = "5f22cd4e6fa16729e946cbdfbacfa27692f35ab8" }
4 changes: 0 additions & 4 deletions datafusion/common/src/scalar.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2979,10 +2979,6 @@ mod tests {
ScalarValue::Decimal128(None, 10, 2),
ScalarValue::try_from_array(&array, 3).unwrap()
);
assert_eq!(
Copy link
Contributor Author

@tustvold tustvold Mar 1, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test was wrong, index 4 is beyond the bounds of the array. Previously this was fine as the null buffer only enforced the length for multiples of 8

ScalarValue::Decimal128(None, 10, 2),
ScalarValue::try_from_array(&array, 4).unwrap()
);

Ok(())
}
Expand Down
8 changes: 5 additions & 3 deletions datafusion/physical-expr/src/expressions/in_list.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,9 +162,11 @@ where
}
};

match data.null_buffer() {
Some(buffer) => BitIndexIterator::new(buffer.as_ref(), data.offset(), data.len())
.for_each(insert_value),
match data.nulls() {
Some(nulls) => {
BitIndexIterator::new(nulls.validity(), nulls.offset(), nulls.len())
.for_each(insert_value)
}
None => (0..data.len()).for_each(insert_value),
}

Expand Down
5 changes: 1 addition & 4 deletions datafusion/physical-expr/src/regex_expressions.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,10 +265,7 @@ fn _regexp_replace_static_pattern_replace<T: OffsetSizeTrait>(
let data = ArrayData::try_new(
GenericStringArray::<T>::DATA_TYPE,
string_array.len(),
string_array
.data_ref()
.null_buffer()
.map(|b| b.bit_slice(string_array.offset(), string_array.len())),
string_array.data().nulls().map(|b| b.inner().sliced()),
0,
vec![new_offsets.finish(), vals.finish()],
vec![],
Expand Down
2 changes: 1 addition & 1 deletion datafusion/proto/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,4 @@ tokio = "1.18"
[build-dependencies]
# Pin these dependencies so that the generated output is deterministic
pbjson-build = { version = "=0.5.1" }
prost-build = { version = "=0.11.7" }
prost-build = { version = "=0.11.8" }