diff --git a/Cargo.toml b/Cargo.toml index 9e7971bdc1e8..c2904da3c106 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,7 +125,7 @@ rand = "0.8" regex = "1.8" rstest = "0.21.0" serde_json = "1" -sqlparser = { version = "0.48", features = ["visitor"] } +sqlparser = { version = "0.49", features = ["visitor"] } tempfile = "3" thiserror = "1.0.44" tokio = { version = "1.36", features = ["macros", "rt", "sync"] } diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index a4e87f99b5c3..3dc1db2c9036 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -363,9 +363,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" dependencies = [ "bzip2", "flate2", @@ -387,7 +387,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1104,7 +1104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1162,7 +1162,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap 2.2.6", - "itertools", + "itertools 0.12.1", "log", "num-traits", "num_cpus", @@ -1288,7 +1288,7 @@ dependencies = [ "datafusion-expr", "hashbrown 0.14.5", "hex", - "itertools", + "itertools 0.12.1", "log", "md-5", "rand", @@ -1328,7 +1328,7 @@ dependencies = [ "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", - "itertools", + "itertools 0.12.1", "log", "paste", "rand", @@ -1346,7 +1346,7 @@ dependencies = [ "datafusion-physical-expr", "hashbrown 0.14.5", "indexmap 2.2.6", - "itertools", + "itertools 0.12.1", "log", "paste", "regex-syntax", @@ -1373,7 +1373,7 @@ dependencies = [ "hashbrown 0.14.5", "hex", "indexmap 2.2.6", - "itertools", + "itertools 0.12.1", "log", "paste", "petgraph", @@ -1425,7 +1425,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap 2.2.6", - "itertools", + "itertools 0.12.1", "log", "once_cell", "parking_lot", @@ -1705,7 +1705,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -2027,7 +2027,7 @@ dependencies = [ "http 1.1.0", "hyper 1.4.1", "hyper-util", - "rustls 0.23.11", + "rustls 0.23.12", "rustls-native-certs 0.7.1", "rustls-pki-types", "tokio", @@ -2141,6 +2141,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -2149,9 +2158,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] @@ -2375,13 +2384,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ + "hermit-abi 0.3.9", "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2502,18 +2512,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.1" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" +checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbebfd32c213ba1907fa7a9c9138015a8de2b43e30c5aa45b18f7deb46786ad6" +checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" dependencies = [ "async-trait", "base64 0.22.1", @@ -2522,7 +2532,7 @@ dependencies = [ "futures", "humantime", "hyper 1.4.1", - "itertools", + "itertools 0.13.0", "md-5", "parking_lot", "percent-encoding", @@ -2718,7 +2728,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -2822,9 +2832,9 @@ checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" [[package]] name = "quick-xml" -version = "0.31.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" dependencies = [ "memchr", "serde", @@ -2841,7 +2851,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.11", + "rustls 0.23.12", "thiserror", "tokio", "tracing", @@ -2857,7 +2867,7 @@ dependencies = [ "rand", "ring 0.17.8", "rustc-hash", - "rustls 0.23.11", + "rustls 0.23.12", "slab", "thiserror", "tinyvec", @@ -2866,14 +2876,13 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9096629c45860fc7fb143e125eb826b5e721e10be3263160c7d60ca832cf8c46" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" dependencies = [ "libc", "once_cell", "socket2", - "tracing", "windows-sys 0.52.0", ] @@ -3006,7 +3015,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.11", + "rustls 0.23.12", "rustls-native-certs 0.7.1", "rustls-pemfile 2.1.2", "rustls-pki-types", @@ -3136,9 +3145,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.11" +version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ "once_cell", "ring 0.17.8", @@ -3200,9 +3209,9 @@ checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" -version = "0.102.5" +version = "0.102.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" dependencies = [ "ring 0.17.8", "rustls-pki-types", @@ -3330,7 +3339,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -3449,9 +3458,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.48.0" +version = "0.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "749780d15ad1ee15fd74f5f84b0665560b6abb913de744c2b69155770f9601da" +checksum = "a4a404d0e14905361b918cb8afdb73605e25c1d5029312bd9785142dcb3aa49e" dependencies = [ "log", "sqlparser_derive", @@ -3465,7 +3474,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -3511,7 +3520,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -3524,7 +3533,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -3546,9 +3555,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.71" +version = "2.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" dependencies = [ "proc-macro2", "quote", @@ -3611,7 +3620,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -3681,32 +3690,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.1" +version = "1.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" +checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -3726,7 +3734,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.11", + "rustls 0.23.12", "rustls-pki-types", "tokio", ] @@ -3803,7 +3811,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -3848,7 +3856,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -4002,7 +4010,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", "wasm-bindgen-shared", ] @@ -4036,7 +4044,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4301,7 +4309,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] diff --git a/datafusion/sql/src/expr/order_by.rs b/datafusion/sql/src/expr/order_by.rs index 6010da6fd325..7fb32f714cfa 100644 --- a/datafusion/sql/src/expr/order_by.rs +++ b/datafusion/sql/src/expr/order_by.rs @@ -16,7 +16,9 @@ // under the License. use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; -use datafusion_common::{plan_datafusion_err, plan_err, Column, DFSchema, Result}; +use datafusion_common::{ + not_impl_err, plan_datafusion_err, plan_err, Column, DFSchema, Result, +}; use datafusion_expr::expr::Sort; use datafusion_expr::Expr; use sqlparser::ast::{Expr as SQLExpr, OrderByExpr, Value}; @@ -63,8 +65,13 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { asc, expr, nulls_first, + with_fill, } = e; + if let Some(with_fill) = with_fill { + return not_impl_err!("ORDER BY WITH FILL is not supported: {with_fill}"); + } + let expr = match expr { SQLExpr::Value(Value::Number(v, _)) if literal_to_column => { let field_index = v diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs index 5a774a2397b3..40dd368f9e80 100644 --- a/datafusion/sql/src/parser.rs +++ b/datafusion/sql/src/parser.rs @@ -625,6 +625,7 @@ impl<'a> DFParser<'a> { expr, asc, nulls_first, + with_fill: None, }) } @@ -1206,6 +1207,7 @@ mod tests { }), asc, nulls_first, + with_fill: None, }]], if_not_exists: false, unbounded: false, @@ -1235,6 +1237,7 @@ mod tests { }), asc: Some(true), nulls_first: None, + with_fill: None, }, OrderByExpr { expr: Identifier(Ident { @@ -1243,6 +1246,7 @@ mod tests { }), asc: Some(false), nulls_first: Some(true), + with_fill: None, }, ]], if_not_exists: false, @@ -1278,6 +1282,7 @@ mod tests { }, asc: Some(true), nulls_first: None, + with_fill: None, }]], if_not_exists: false, unbounded: false, @@ -1321,6 +1326,7 @@ mod tests { }, asc: Some(true), nulls_first: None, + with_fill: None, }]], if_not_exists: true, unbounded: true, diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rs index 00560b5c9308..ba2b41bb6ecf 100644 --- a/datafusion/sql/src/query.rs +++ b/datafusion/sql/src/query.rs @@ -19,13 +19,14 @@ use std::sync::Arc; use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; -use datafusion_common::{plan_err, Constraints, Result, ScalarValue}; +use datafusion_common::{not_impl_err, plan_err, Constraints, Result, ScalarValue}; use datafusion_expr::{ CreateMemoryTable, DdlStatement, Distinct, Expr, LogicalPlan, LogicalPlanBuilder, Operator, }; use sqlparser::ast::{ - Expr as SQLExpr, Offset as SQLOffset, Query, SelectInto, SetExpr, Value, + Expr as SQLExpr, Offset as SQLOffset, OrderBy, OrderByExpr, Query, SelectInto, + SetExpr, Value, }; impl<'a, S: ContextProvider> SqlToRel<'a, S> { @@ -50,16 +51,17 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { let select_into = select.into.take(); // Order-by expressions may refer to columns in the `FROM` clause, // so we need to process `SELECT` and `ORDER BY` together. - let plan = - self.select_to_plan(*select, query.order_by, planner_context)?; + let oby_exprs = to_order_by_exprs(query.order_by)?; + let plan = self.select_to_plan(*select, oby_exprs, planner_context)?; let plan = self.limit(plan, query.offset, query.limit)?; // Process the `SELECT INTO` after `LIMIT`. self.select_into(plan, select_into) } other => { let plan = self.set_expr_to_plan(other, planner_context)?; + let oby_exprs = to_order_by_exprs(query.order_by)?; let order_by_rex = self.order_by_to_sort_expr( - query.order_by, + oby_exprs, plan.schema(), planner_context, true, @@ -198,3 +200,15 @@ fn convert_usize_with_check(n: i64, arg_name: &str) -> Result { Ok(n as usize) } } + +/// Returns the order by expressions from the query. +fn to_order_by_exprs(order_by: Option) -> Result> { + let Some(OrderBy { exprs, interpolate }) = order_by else { + // if no order by, return an empty array + return Ok(vec![]); + }; + if let Some(_interpolate) = interpolate { + return not_impl_err!("ORDER BY INTERPOLATE is not supported"); + } + Ok(exprs) +} diff --git a/datafusion/sql/src/relation/mod.rs b/datafusion/sql/src/relation/mod.rs index b812dae5ae3e..5d7b3d5918d3 100644 --- a/datafusion/sql/src/relation/mod.rs +++ b/datafusion/sql/src/relation/mod.rs @@ -101,7 +101,12 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { array_exprs, with_offset: false, with_offset_alias: None, + with_ordinality, } => { + if with_ordinality { + return not_impl_err!("UNNEST with ordinality is not supported yet"); + } + // Unnest table factor has empty input let schema = DFSchema::empty(); let input = LogicalPlanBuilder::empty(true).build()?; diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 67107bae0202..d5e33f4849a4 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -171,7 +171,10 @@ fn calc_inline_constraints_from_columns(columns: &[ColumnDef]) -> Vec {} + | ast::ColumnOption::OnUpdate(_) + | ast::ColumnOption::Materialized(_) + | ast::ColumnOption::Ephemeral(_) + | ast::ColumnOption::Alias(_) => {} } } } diff --git a/datafusion/sql/src/unparser/ast.rs b/datafusion/sql/src/unparser/ast.rs index 02eb44dbb657..c10db9831457 100644 --- a/datafusion/sql/src/unparser/ast.rs +++ b/datafusion/sql/src/unparser/ast.rs @@ -80,13 +80,22 @@ impl QueryBuilder { self } pub fn build(&self) -> Result { + let order_by = if self.order_by.is_empty() { + None + } else { + Some(ast::OrderBy { + exprs: self.order_by.clone(), + interpolate: None, + }) + }; + Ok(ast::Query { with: self.with.clone(), body: match self.body { Some(ref value) => value.clone(), None => return Err(Into::into(UninitializedFieldError::from("body"))), }, - order_by: self.order_by.clone(), + order_by, limit: self.limit.clone(), limit_by: self.limit_by.clone(), offset: self.offset.clone(), @@ -423,6 +432,7 @@ impl TableRelationBuilder { with_hints: self.with_hints.clone(), version: self.version.clone(), partitions: self.partitions.clone(), + with_ordinality: false, }) } fn create_empty() -> Self { diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs index 3f7a85da276b..e144dfd649d2 100644 --- a/datafusion/sql/src/unparser/expr.rs +++ b/datafusion/sql/src/unparser/expr.rs @@ -543,6 +543,7 @@ impl Unparser<'_> { expr: sql_parser_expr, asc: Some(*asc), nulls_first, + with_fill: None, })) } _ => { diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs index 59660f4f0404..9519b520ce5e 100644 --- a/datafusion/sql/src/unparser/plan.rs +++ b/datafusion/sql/src/unparser/plan.rs @@ -558,6 +558,7 @@ impl Unparser<'_> { asc: Some(sort_expr.asc), expr: col, nulls_first, + with_fill: None, }) } _ => plan_err!("Expecting Sort expr"), diff --git a/dev/depcheck/Cargo.toml b/dev/depcheck/Cargo.toml index cb4e77eabb22..23cefaec43be 100644 --- a/dev/depcheck/Cargo.toml +++ b/dev/depcheck/Cargo.toml @@ -22,4 +22,4 @@ name = "depcheck" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -cargo = "0.78.1" +cargo = "0.81.0" diff --git a/dev/depcheck/src/main.rs b/dev/depcheck/src/main.rs index 1599fdd4188d..80feefcd1b1c 100644 --- a/dev/depcheck/src/main.rs +++ b/dev/depcheck/src/main.rs @@ -23,7 +23,7 @@ use std::collections::{HashMap, HashSet}; use std::env; use std::path::Path; -use cargo::util::config::Config; +use cargo::util::context::GlobalContext; /// Verifies that there are no circular dependencies between DataFusion crates /// (which prevents publishing on crates.io) by parsing the Cargo.toml files and @@ -31,7 +31,7 @@ use cargo::util::config::Config; /// /// See https://github.com/apache/datafusion/issues/9278 for more details fn main() -> CargoResult<()> { - let config = Config::default()?; + let gctx = GlobalContext::default()?; // This is the path for the depcheck binary let path = env::var("CARGO_MANIFEST_DIR").unwrap(); let root_cargo_toml = Path::new(&path) @@ -47,7 +47,7 @@ fn main() -> CargoResult<()> { "Checking for circular dependencies in {}", root_cargo_toml.display() ); - let workspace = cargo::core::Workspace::new(&root_cargo_toml, &config)?; + let workspace = cargo::core::Workspace::new(&root_cargo_toml, &gctx)?; let (_, resolve) = cargo::ops::resolve_ws(&workspace)?; let mut package_deps = HashMap::new();