Merge remote-tracking branch 'apache/main' into datafusion_catalog_li…

…sting
logan-keede · Feb 5, 2025 · 2080d17 · 2080d17
2 parents 9fd1eb3 + 168fe49
commit 2080d17
Show file tree

Hide file tree

Showing 35 changed files with 815 additions and 210 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -4,7 +4,7 @@
 We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123.
 -->
 
-Closes #.
+- Closes #.
 
 ## Rationale for this change
 

diff --git a/Cargo.toml b/Cargo.toml
@@ -95,7 +95,7 @@ arrow-ord = { version = "54.1.0", default-features = false }
 arrow-schema = { version = "54.1.0", default-features = false }
 async-trait = "0.1.73"
 bigdecimal = "0.4.7"
-bytes = "1.4"
+bytes = "1.10"
 chrono = { version = "0.4.38", default-features = false }
 ctor = "0.2.9"
 dashmap = "6.0.1"
@@ -152,7 +152,7 @@ rstest = "0.24.0"
 serde_json = "1"
 sqlparser = { version = "0.53.0", features = ["visitor"] }
 tempfile = "3"
-tokio = { version = "1.36", features = ["macros", "rt", "sync"] }
+tokio = { version = "1.43", features = ["macros", "rt", "sync"] }
 url = "2.5.4"
 
 [profile.release]

diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
@@ -31,12 +31,9 @@ readme = "README.md"
 [dependencies]
 arrow = { version = "54.1.0" }
 async-trait = "0.1.0"
-aws-config = "1.5.0"
+aws-config = "1.5.16"
 aws-credential-types = "1.2.0"
-aws-sdk-sso = "1.57.0"
-aws-sdk-ssooidc = "1.57.0"
-aws-sdk-sts = "1.57.0"
-clap = { version = "4.5.27", features = ["derive", "cargo"] }
+clap = { version = "4.5.28", features = ["derive", "cargo"] }
 datafusion = { path = "../datafusion/core", version = "45.0.0", features = [
     "avro",
     "crypto_expressions",
@@ -52,14 +49,13 @@ dirs = "6.0.0"
 env_logger = "0.11"
 futures = "0.3"
 # pin as home 0.5.11 has MSRV 1.81. Can remove this once we bump MSRV to 1.81
-home = "=0.5.11"
 mimalloc = { version = "0.1", default-features = false }
 object_store = { version = "0.11.0", features = ["aws", "gcp", "http"] }
 parking_lot = { version = "0.12" }
 parquet = { version = "54.1.0", default-features = false }
 regex = "1.8"
 rustyline = "15.0"
-tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot", "signal"] }
+tokio = { version = "1.43", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot", "signal"] }
 url = "2.5.4"
 
 [dev-dependencies]

diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
@@ -132,7 +132,7 @@ tempfile = { workspace = true }
 tokio = { workspace = true }
 tokio-util = { version = "0.7.4", features = ["io"], optional = true }
 url = { workspace = true }
-uuid = { version = "1.7", features = ["v4"] }
+uuid = { version = "1.7", features = ["v4", "js"] }
 xz2 = { version = "0.1", optional = true, features = ["static"] }
 zstd = { version = "0.13", optional = true, default-features = false }
 

diff --git a/datafusion/core/benches/aggregate_query_sql.rs b/datafusion/core/benches/aggregate_query_sql.rs
@@ -173,6 +173,39 @@ fn criterion_benchmark(c: &mut Criterion) {
             )
         })
     });
+
+    c.bench_function("first_last_many_columns", |b| {
+        b.iter(|| {
+            query(
+                ctx.clone(),
+                "SELECT first_value(u64_wide order by f64, u64_narrow, utf8),\
+                            last_value(u64_wide order by f64, u64_narrow, utf8)  \
+                 FROM t GROUP BY u64_narrow",
+            )
+        })
+    });
+
+    c.bench_function("first_last_ignore_nulls", |b| {
+        b.iter(|| {
+            query(
+                ctx.clone(),
+                "SELECT first_value(u64_wide ignore nulls order by f64, u64_narrow, utf8),  \
+                            last_value(u64_wide ignore nulls order by f64, u64_narrow, utf8)    \
+                 FROM t GROUP BY u64_narrow",
+            )
+        })
+    });
+
+    c.bench_function("first_last_one_column", |b| {
+        b.iter(|| {
+            query(
+                ctx.clone(),
+                "SELECT first_value(u64_wide order by f64), \
+                            last_value(u64_wide order by f64)   \
+                FROM t GROUP BY u64_narrow",
+            )
+        })
+    });
 }
 
 criterion_group!(benches, criterion_benchmark);

diff --git a/datafusion/core/src/datasource/physical_plan/parquet/mod.rs b/datafusion/core/src/datasource/physical_plan/parquet/mod.rs
@@ -32,10 +32,7 @@ use std::sync::Arc;
 
 use crate::datasource::listing::PartitionedFile;
 use crate::datasource::physical_plan::file_stream::FileStream;
-use crate::datasource::physical_plan::{
-    parquet::page_filter::PagePruningAccessPlanFilter, DisplayAs, FileGroupPartitioner,
-    FileScanConfig,
-};
+use crate::datasource::physical_plan::{DisplayAs, FileGroupPartitioner, FileScanConfig};
 use crate::datasource::schema_adapter::{
     DefaultSchemaAdapterFactory, SchemaAdapterFactory,
 };
@@ -58,8 +55,10 @@ use datafusion_physical_optimizer::pruning::PruningPredicate;
 use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
 pub use metrics::ParquetFileMetrics;
 use opener::ParquetOpener;
+pub use page_filter::PagePruningAccessPlanFilter;
 pub use reader::{DefaultParquetFileReaderFactory, ParquetFileReaderFactory};
 pub use row_filter::can_expr_be_pushed_down_with_schemas;
+pub use row_group_filter::RowGroupAccessPlanFilter;
 pub use writer::plan_to_parquet;
 
 use itertools::Itertools;