diff --git a/Cargo.lock b/Cargo.lock index 8d942d761..d37844800 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -553,6 +553,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atoi_simd" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc41b65e01b6851bdcd2d741824e6b310d571396bf3915e31e4792034ee65126" + [[package]] name = "autocfg" version = "1.1.0" @@ -1542,15 +1548,16 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" +checksum = "1f628eaec48bfd21b865dc2950cfa014450c01d2fa2b69a86c2fd5844ec523c0" dependencies = [ "curve25519-dalek", "ed25519", "serde", "sha2", "signature", + "subtle", "zeroize", ] @@ -1695,9 +1702,9 @@ checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fiat-crypto" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a56f0780318174bad1c127063fd0c5fdfb35398e3cd79ffaab931a6c79df80" +checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7" [[package]] name = "file-format" @@ -2544,15 +2551,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760" -[[package]] -name = "lexical" -version = "6.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7aefb36fd43fef7003334742cbf77b243fcd36418a1d1bdd480d613a67968f6" -dependencies = [ - "lexical-core", -] - [[package]] name = "lexical-core" version = "0.8.5" @@ -3180,25 +3178,6 @@ dependencies = [ "futures", ] -[[package]] -name = "parquet2" -version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "579fe5745f02cef3d5f236bfed216fd4693e49e4e920a13475c6132233283bce" -dependencies = [ - "async-stream", - "brotli", - "flate2", - "futures", - "lz4", - "parquet-format-safe", - "seq-macro", - "snap", - "streaming-decompression", - "xxhash-rust", - "zstd 0.12.4", -] - [[package]] name = "password-hash" version = "0.4.2" @@ -3419,9 +3398,9 @@ checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0" [[package]] name = "polars" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40db657cc67a8dd9fe4b40db5b73027f5f224623545597e1930cbbb9c05b1de5" +checksum = "987f70ba422ed3f6bb566d4a02f5ad35f2cabcdc7f09705f9340a5ee2ad59215" dependencies = [ "getrandom", "polars-core", @@ -3435,30 +3414,31 @@ dependencies = [ [[package]] name = "polars-arrow" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1e50c63db77f846ac5119477422f0156f0a1826ceaae7d921f9a6d5ea5f7ca3" +checksum = "682c24cbb6fc05882c0112da75ec037d0afe88241c3778c074d61211bbf063bf" dependencies = [ "ahash 0.8.6", "arrow-format", - "base64", + "atoi_simd", "bytemuck", "chrono", "dyn-clone", "either", "ethnum", - "fallible-streaming-iterator", + "fast-float", "foreign_vec", "futures", "getrandom", "hashbrown 0.14.2", - "lexical-core", + "itoa", "lz4", "multiversion", "num-traits", - "parquet2", "polars-error", + "polars-utils", "rustc_version", + "ryu", "simdutf8", "streaming-iterator", "strength_reduce", @@ -3467,9 +3447,9 @@ dependencies = [ [[package]] name = "polars-core" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdfb622b8ca81b4614c64d95e7590d6e0571d7d398b5ad595c1abc4412abe714" +checksum = "0fcbe0039cad625b5775c8f8e0dfdd204a71968959ae006322083b3c244475bf" dependencies = [ "ahash 0.8.6", "bitflags 2.4.1", @@ -3498,12 +3478,11 @@ dependencies = [ [[package]] name = "polars-error" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6480520ebde0b20935b600483b865513891e36c04942cebdd19e4f338257b4" +checksum = "d1e73784f0357111c71a6f0f765ec91a5f1487bc5cdd9d41e54f81fd53e06e73" dependencies = [ "arrow-format", - "parquet2", "regex", "simdutf8", "thiserror", @@ -3511,20 +3490,19 @@ dependencies = [ [[package]] name = "polars-io" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666466a3b151047c76d99b4e4e5f5438895ef97848008cf49b06df8e3d2d395a" +checksum = "9080fcc2777fcdfb859cfd45029a90096089bd0da15f4f022c23f8276bf5268c" dependencies = [ "ahash 0.8.6", "async-trait", + "atoi_simd", "bytes", "chrono", "fast-float", "futures", "home", "itoa", - "lexical", - "lexical-core", "memchr", "memmap2", "num-traits", @@ -3534,6 +3512,7 @@ dependencies = [ "polars-core", "polars-error", "polars-json", + "polars-parquet", "polars-time", "polars-utils", "rayon", @@ -3549,9 +3528,9 @@ dependencies = [ [[package]] name = "polars-json" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24451d2647a9bd51283cc946509c23bac27130565daa5103a156c8507b85b5a3" +checksum = "4d90ab8b514897140841dcf337810d1ea5778ac260fd561896365445e595275c" dependencies = [ "ahash 0.8.6", "chrono", @@ -3570,9 +3549,9 @@ dependencies = [ [[package]] name = "polars-lazy" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e1c2da1ca20106f80d9510090344e7311fd1dcfd6e6b65031e10606c0958c7" +checksum = "f20c79e04461bc3fa085c552c7c18919732ab6cde4e5da9f7b52b48d269b0f11" dependencies = [ "ahash 0.8.6", "bitflags 2.4.1", @@ -3594,9 +3573,9 @@ dependencies = [ [[package]] name = "polars-ops" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe2d37a6a3ef358499d43aecee80740e62dd44e6cfe7a9c4aa0b8db88de8292" +checksum = "82ad422e9bb854d275f8d700e0f2bbf39cd8f71709a24aa40e54f32ac7410199" dependencies = [ "ahash 0.8.6", "argminmax", @@ -3616,11 +3595,37 @@ dependencies = [ "version_check", ] +[[package]] +name = "polars-parquet" +version = "0.35.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d73479f136c036d1ba87d7ea2e9ac89c86ee2c16ab0020a6e15edbb3581135" +dependencies = [ + "ahash 0.8.6", + "async-stream", + "base64", + "brotli", + "ethnum", + "flate2", + "futures", + "lz4", + "num-traits", + "parquet-format-safe", + "polars-arrow", + "polars-error", + "polars-utils", + "seq-macro", + "simdutf8", + "snap", + "streaming-decompression", + "zstd 0.13.0", +] + [[package]] name = "polars-pipe" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6aa050d529be01617f54bc60658149da76f97dbea9fdac3c9d60b811f64a2ba" +checksum = "869f372f023554f8a66fc06060709993b92004f2157188d467741c8fcdb226ff" dependencies = [ "crossbeam-channel", "crossbeam-queue", @@ -3641,9 +3646,9 @@ dependencies = [ [[package]] name = "polars-plan" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c47e5d62d8f612aab61a6331d04c5c95c9ff301106d8b91131c8833b4ef3def6" +checksum = "af139995be30bb6469fdff78669be5da6aad9ae7ec1ec3e538f34ebeaa8b0fb4" dependencies = [ "ahash 0.8.6", "bytemuck", @@ -3653,6 +3658,7 @@ dependencies = [ "polars-core", "polars-io", "polars-ops", + "polars-parquet", "polars-time", "polars-utils", "rayon", @@ -3664,9 +3670,9 @@ dependencies = [ [[package]] name = "polars-row" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05d6544f7d6065fcaa93bc69aac0532ce09aab4f81ec03c9a78dd901bb0c05b" +checksum = "7ae5284d603c3111c217d2451e1b62578eebf2f30d49c32e03790dd4f9cc21ad" dependencies = [ "polars-arrow", "polars-error", @@ -3675,9 +3681,9 @@ dependencies = [ [[package]] name = "polars-sql" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f65f9c8bfe7f0b2c08c38c79b92ec4ddaf213fc424d94a6272ed7b2d83987f" +checksum = "9357cc60db55c2a407230b42e3ca67ef8495513460ade6f368f48fdd4d2a2505" dependencies = [ "polars-arrow", "polars-core", @@ -3692,9 +3698,9 @@ dependencies = [ [[package]] name = "polars-time" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3763af36aeeb85ef083f11c43bc28c5b6222e2aae039c5118d916bc855f2b5b9" +checksum = "cdbd4ad82122f9fa8bebfc40dc41183e311b0a8ce2e58f2a48800b5e1136d41e" dependencies = [ "atoi", "chrono", @@ -3711,13 +3717,14 @@ dependencies = [ [[package]] name = "polars-utils" -version = "0.34.2" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55d2c038ff67e4eb6019682c3f66d83f744e285de9c28e816109a61bace824cd" +checksum = "3075dda9bfecbdcd3d721c6ed4f15f00edf58acba3d5e7711e6f1a9cedf46444" dependencies = [ "ahash 0.8.6", "bytemuck", "hashbrown 0.14.2", + "indexmap 2.1.0", "num-traits", "once_cell", "polars-error", @@ -4296,6 +4303,26 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "ref-cast" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acde58d073e9c79da00f2b5b84eed919c8326832648a5b109b3fce1bb1175280" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "regex" version = "1.10.2" @@ -4515,9 +4542,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.8" +version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", "ring", @@ -4797,24 +4824,26 @@ dependencies = [ [[package]] name = "signature" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fe458c98333f9c8152221191a77e2a44e8325d0193484af2e9421a53019e57d" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest", + "rand_core", ] [[package]] name = "simd-json" -version = "0.12.0" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f07a84c7456b901b8dd2c1d44caca8b0fd2c2616206ee5acc9d9da61e8d9ec" +checksum = "e5a3720326b20bf5b95b72dbbd133caae7e0dcf71eae8f6e6656e71a7e5c9aaa" dependencies = [ "ahash 0.8.6", "getrandom", "halfbrown", "lexical-core", "once_cell", + "ref-cast", "serde", "serde_json", "simdutf8", @@ -4937,9 +4966,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.38.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272b7bb0a225320170c99901b4b5fb3a4384e255a7f2cc228f61e2ba3893e75" +checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7" dependencies = [ "log", ] @@ -5577,9 +5606,9 @@ dependencies = [ [[package]] name = "value-trait" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a5b6c8ceb01263b969cac48d4a6705134d490ded13d889e52c0cfc80c6945e" +checksum = "ea87257cfcbedcb9444eda79c59fdfea71217e6305afee8ee33f500375c2ac97" dependencies = [ "float-cmp", "halfbrown", @@ -6007,9 +6036,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12a3946ecfc929b583800f4629b6c25b88ac6e92a40ea5670f77112a85d40a8b" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" [[package]] name = "zip" @@ -6050,15 +6079,6 @@ dependencies = [ "zstd-safe 5.0.2+zstd.1.5.2", ] -[[package]] -name = "zstd" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" -dependencies = [ - "zstd-safe 6.0.6", -] - [[package]] name = "zstd" version = "0.13.0" @@ -6078,16 +6098,6 @@ dependencies = [ "zstd-sys", ] -[[package]] -name = "zstd-safe" -version = "6.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" -dependencies = [ - "libc", - "zstd-sys", -] - [[package]] name = "zstd-safe" version = "7.0.0" diff --git a/Cargo.toml b/Cargo.toml index c1e66c1fb..198fb198c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -137,7 +137,7 @@ mlua = { version = "0.9", features = [ num_cpus = "1" odht = "0.3" phf = { version = "0.11", features = ["macros"], optional = true } -polars = { version = "0.34", features = [ +polars = { version = "0.35", features = [ "lazy", "streaming", "object", @@ -150,6 +150,7 @@ polars = { version = "0.34", features = [ "ipc", "performant", "cse", + "dtype-categorical" ], optional = true } pyo3 = { version = "0.20", features = ["auto-initialize"], optional = true } qsv-dateparser = "0.10" diff --git a/src/cmd/joinp.rs b/src/cmd/joinp.rs index 3b2124d27..d89db91cc 100644 --- a/src/cmd/joinp.rs +++ b/src/cmd/joinp.rs @@ -451,7 +451,7 @@ impl JoinStruct { let join_shape = results_df.shape(); CsvWriter::new(&mut out_writer) - .has_header(true) + .include_header(true) .with_separator(self.delim) .with_datetime_format(self.datetime_format) .with_date_format(self.date_format) diff --git a/src/cmd/sqlp.rs b/src/cmd/sqlp.rs index 569421ad3..d5fe14f15 100644 --- a/src/cmd/sqlp.rs +++ b/src/cmd/sqlp.rs @@ -6,8 +6,8 @@ Polars SQL is a SQL dialect, converting SQL queries to fast Polars LazyFrame exp (see https://pola-rs.github.io/polars-book/user-guide/sql/intro/) For a list of SQL functions and keywords supported by Polars SQL, see -https://github.com/pola-rs/polars/blob/rs-0.34.0/crates/polars-sql/src/functions.rs -https://github.com/pola-rs/polars/blob/rs-0.34.0/crates/polars-sql/src/keywords.rs and +https://github.com/pola-rs/polars/blob/rs-0.35.0/crates/polars-sql/src/functions.rs +https://github.com/pola-rs/polars/blob/rs-0.35.0/crates/polars-sql/src/keywords.rs and https://github.com/pola-rs/polars/issues/7227 Returns the shape of the query result (number of rows, number of columns) to stderr. @@ -23,6 +23,8 @@ Example queries: qsv sqlp data.csv data2.csv 'select * from data join data2 on data.colname = data2.colname' + qsv sqlp data.csv data2.csv 'SELECT col1 FROM data WHERE col1 IN (SELECT col2 FROM data2)' + qsv sqlp data1.csv data2.csv 'SELECT * FROM data1 UNION ALL BY NAME SELECT * FROM data2' qsv sqlp tbl_a.csv tbl_b.csv tbl_c.csv "SELECT * FROM tbl_a \