Updating polars and sqlparser versions (#10114)

Polars and SQLParser upgrade.

I have exposed features that have been added to polars as command args
where appropriate.

---------

Co-authored-by: Jack Wright <jack.wright@disqo.com>
Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com>
Co-authored-by: sholderbach <sholderbach@users.noreply.github.com>
This commit is contained in:
Jack Wright 2023-08-29 15:13:34 -07:00 committed by GitHub
parent 3f2c76df28
commit 3fd1a26ec0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 236 additions and 236 deletions

185
Cargo.lock generated
View file

@ -166,9 +166,9 @@ dependencies = [
[[package]] [[package]]
name = "arrow2" name = "arrow2"
version = "0.17.2" version = "0.17.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15ae0428d69ab31d7b2adad22a752d6f11fef2e901d2262d0cad4f5cb08b7093" checksum = "59c468daea140b747d781a1da9f7db5f0a8e6636d4af20cc539e43d05b0604fa"
dependencies = [ dependencies = [
"ahash", "ahash",
"arrow-format", "arrow-format",
@ -370,6 +370,9 @@ name = "bitflags"
version = "2.3.3" version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "block-buffer" name = "block-buffer"
@ -679,9 +682,9 @@ dependencies = [
[[package]] [[package]]
name = "comfy-table" name = "comfy-table"
version = "6.2.0" version = "7.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e959d788268e3bf9d35ace83e81b124190378e4c91c9067524675e33394b8ba" checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b"
dependencies = [ dependencies = [
"crossterm", "crossterm",
"strum 0.24.1", "strum 0.24.1",
@ -888,7 +891,7 @@ dependencies = [
"crossterm_winapi", "crossterm_winapi",
"libc", "libc",
"mio", "mio",
"parking_lot 0.12.1", "parking_lot",
"serde", "serde",
"signal-hook", "signal-hook",
"signal-hook-mio", "signal-hook-mio",
@ -984,7 +987,7 @@ dependencies = [
"hashbrown 0.14.0", "hashbrown 0.14.0",
"lock_api", "lock_api",
"once_cell", "once_cell",
"parking_lot_core 0.9.8", "parking_lot_core",
] ]
[[package]] [[package]]
@ -1597,7 +1600,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
dependencies = [ dependencies = [
"ahash", "ahash",
"rayon",
] ]
[[package]] [[package]]
@ -1608,6 +1610,7 @@ checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
dependencies = [ dependencies = [
"ahash", "ahash",
"allocator-api2", "allocator-api2",
"rayon",
] ]
[[package]] [[package]]
@ -1793,7 +1796,6 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"hashbrown 0.12.3", "hashbrown 0.12.3",
"serde",
] ]
[[package]] [[package]]
@ -1804,6 +1806,7 @@ checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
dependencies = [ dependencies = [
"equivalent", "equivalent",
"hashbrown 0.14.0", "hashbrown 0.14.0",
"serde",
] ]
[[package]] [[package]]
@ -2310,9 +2313,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]] [[package]]
name = "memmap2" name = "memmap2"
version = "0.5.10" version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6"
dependencies = [ dependencies = [
"libc", "libc",
] ]
@ -2429,9 +2432,9 @@ dependencies = [
[[package]] [[package]]
name = "multiversion" name = "multiversion"
version = "0.7.2" version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cda45dade5144c2c929bf2ed6c24bebbba784e9198df049ec87d722b9462bd1" checksum = "b2c7b9d7fe61760ce5ea19532ead98541f6b4c495d87247aff9826445cf6872a"
dependencies = [ dependencies = [
"multiversion-macros", "multiversion-macros",
"target-features", "target-features",
@ -2439,9 +2442,9 @@ dependencies = [
[[package]] [[package]]
name = "multiversion-macros" name = "multiversion-macros"
version = "0.7.2" version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04bffdccbd4798b61dce08c97ce8c66a68976f95541aaf284a6e90c1d1c306e1" checksum = "26a83d8500ed06d68877e9de1dde76c1dbb83885dcdbda4ef44ccbc3fbda2ac8"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2534,7 +2537,7 @@ checksum = "416969970ec751a5d702a88c6cd19ac1332abe997fce43f96db0418550426241"
dependencies = [ dependencies = [
"file-id", "file-id",
"notify", "notify",
"parking_lot 0.12.1", "parking_lot",
"walkdir", "walkdir",
] ]
@ -2806,7 +2809,6 @@ dependencies = [
"serde_urlencoded", "serde_urlencoded",
"serde_yaml", "serde_yaml",
"sha2", "sha2",
"sqlparser",
"sysinfo", "sysinfo",
"tabled", "tabled",
"terminal_size 0.2.6", "terminal_size 0.2.6",
@ -3371,17 +3373,6 @@ dependencies = [
"unicode-width", "unicode-width",
] ]
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.6",
]
[[package]] [[package]]
name = "parking_lot" name = "parking_lot"
version = "0.12.1" version = "0.12.1"
@ -3389,21 +3380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [ dependencies = [
"lock_api", "lock_api",
"parking_lot_core 0.9.8", "parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall 0.2.16",
"smallvec",
"winapi",
] ]
[[package]] [[package]]
@ -3633,9 +3610,9 @@ dependencies = [
[[package]] [[package]]
name = "polars" name = "polars"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c464be75f4ec8b1a8527a55583f67961e3514f112a99f789476a7ccb9a397d10" checksum = "b1362d4a136c0ebacb40d88a37ba361738b222fd8a2ee9340a3d8642f698c52b"
dependencies = [ dependencies = [
"getrandom", "getrandom",
"polars-core", "polars-core",
@ -3649,33 +3626,34 @@ dependencies = [
[[package]] [[package]]
name = "polars-arrow" name = "polars-arrow"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f187a7fd8fbef04d6a810bfd4c181baf584fc4439ec054e0caad4325bf00f95" checksum = "f967c901fa5da4ca7f64e813d1268488ba97e9b3004cefc579ff851c197a1138"
dependencies = [ dependencies = [
"arrow2", "arrow2",
"hashbrown 0.13.2", "hashbrown 0.14.0",
"multiversion", "multiversion",
"num-traits", "num-traits",
"polars-error", "polars-error",
"serde", "serde",
"thiserror", "thiserror",
"version_check",
] ]
[[package]] [[package]]
name = "polars-core" name = "polars-core"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78b7216aa3336fd2a7b5ebfa66748f3770682b28cd682930d1abf59b53b670e0" checksum = "b24f92fc5b167f668ff85ab9607dfa72e2c09664cacef59297ee8601dee60126"
dependencies = [ dependencies = [
"ahash", "ahash",
"arrow2", "arrow2",
"bitflags 1.3.2", "bitflags 2.3.3",
"chrono", "chrono",
"comfy-table", "comfy-table",
"either", "either",
"hashbrown 0.13.2", "hashbrown 0.14.0",
"indexmap 1.9.3", "indexmap 2.0.0",
"num-traits", "num-traits",
"once_cell", "once_cell",
"polars-arrow", "polars-arrow",
@ -3690,15 +3668,15 @@ dependencies = [
"serde_json", "serde_json",
"smartstring", "smartstring",
"thiserror", "thiserror",
"wasm-timer", "version_check",
"xxhash-rust", "xxhash-rust",
] ]
[[package]] [[package]]
name = "polars-error" name = "polars-error"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfb4f020b2b6d12408cca6b081dbd928fef8bbe9cdbba4c88441e61ade33bc2d" checksum = "40d09c3a7337e53b38c37b57999038440fa39c6801b9ba48afaecd8e16f7ac0a"
dependencies = [ dependencies = [
"arrow2", "arrow2",
"regex", "regex",
@ -3707,9 +3685,9 @@ dependencies = [
[[package]] [[package]]
name = "polars-io" name = "polars-io"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "270650704e08bef37d227a6904b36c4bdad2d013536ea3c57f40007c19ad2ebc" checksum = "92cab0df9f2a35702fa5aec99edfaabf9ae8e9cdd0acf69e143ad2d132f34f9c"
dependencies = [ dependencies = [
"ahash", "ahash",
"arrow2", "arrow2",
@ -3742,15 +3720,15 @@ dependencies = [
[[package]] [[package]]
name = "polars-json" name = "polars-json"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7370203bca4ee29da6da4210200894eea6abf81d7c430e192159dabcba8441a4" checksum = "9e58094557cf6446808c7348dcb797885db61815857f6ea02924b35505566e94"
dependencies = [ dependencies = [
"ahash", "ahash",
"arrow2", "arrow2",
"fallible-streaming-iterator", "fallible-streaming-iterator",
"hashbrown 0.13.2", "hashbrown 0.14.0",
"indexmap 1.9.3", "indexmap 2.0.0",
"num-traits", "num-traits",
"polars-arrow", "polars-arrow",
"polars-error", "polars-error",
@ -3760,12 +3738,12 @@ dependencies = [
[[package]] [[package]]
name = "polars-lazy" name = "polars-lazy"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2a8fe41263496b5212098f19d0cdddc11f75c71957a09d2ce300a8fdb4407e3" checksum = "2c33762ec2a55e01c9f8776b34db86257c70a0a3b3929bd4eb91a52aacf61456"
dependencies = [ dependencies = [
"ahash", "ahash",
"bitflags 1.3.2", "bitflags 2.3.3",
"glob", "glob",
"once_cell", "once_cell",
"polars-arrow", "polars-arrow",
@ -3779,33 +3757,36 @@ dependencies = [
"polars-utils", "polars-utils",
"rayon", "rayon",
"smartstring", "smartstring",
"version_check",
] ]
[[package]] [[package]]
name = "polars-ops" name = "polars-ops"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d63c31565b88f31457abafb224166fdc0824bf2954414f0c5c52b31c09189942" checksum = "e825575c96302d2daedfc205a0062180033c92c55bcd6aafc4e109d4d8849ed0"
dependencies = [ dependencies = [
"argminmax", "argminmax",
"arrow2", "arrow2",
"either", "either",
"indexmap 2.0.0",
"memchr", "memchr",
"polars-arrow", "polars-arrow",
"polars-core", "polars-core",
"polars-utils", "polars-utils",
"serde", "serde",
"smartstring", "smartstring",
"version_check",
] ]
[[package]] [[package]]
name = "polars-pipe" name = "polars-pipe"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51076665e7e45f536a2f080763db040e38eab9fdb1cd927b93428720ba23c46a" checksum = "1f2bc9a12da9ed043fb0cb51dbcb87b365e4845b7ab6399d7a81e838460c6974"
dependencies = [ dependencies = [
"enum_dispatch", "enum_dispatch",
"hashbrown 0.13.2", "hashbrown 0.14.0",
"num-traits", "num-traits",
"polars-arrow", "polars-arrow",
"polars-core", "polars-core",
@ -3816,13 +3797,14 @@ dependencies = [
"polars-utils", "polars-utils",
"rayon", "rayon",
"smartstring", "smartstring",
"version_check",
] ]
[[package]] [[package]]
name = "polars-plan" name = "polars-plan"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a7d0f48bdd7fa9474f718ecb99fc12d97671933707091a249bb0da96b30f291" checksum = "fb67b014f0295e8e9dbb84404a91d666d477b3bc248a2ed51bc442833b16da35"
dependencies = [ dependencies = [
"ahash", "ahash",
"arrow2", "arrow2",
@ -3837,13 +3819,15 @@ dependencies = [
"regex", "regex",
"serde", "serde",
"smartstring", "smartstring",
"strum_macros 0.25.1",
"version_check",
] ]
[[package]] [[package]]
name = "polars-row" name = "polars-row"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b18fa58195b88fdd32c98561cbeff4024b106cdd2a8c975e7d06544324d0e4d1" checksum = "27f54c1956027bf6301948fb4f2837cf6d6b638d8dd1edf3aaeaa19906a986be"
dependencies = [ dependencies = [
"arrow2", "arrow2",
"polars-error", "polars-error",
@ -3852,9 +3836,9 @@ dependencies = [
[[package]] [[package]]
name = "polars-sql" name = "polars-sql"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8485d0c9664e67123b065c8a7c946a0165b0192c05c24cdcde447b1c9e05c7e5" checksum = "dbfcb15cf8eebd25ea1724109d0153817cd484c6326290585f0736b4e7fcf2f4"
dependencies = [ dependencies = [
"polars-arrow", "polars-arrow",
"polars-core", "polars-core",
@ -3867,9 +3851,9 @@ dependencies = [
[[package]] [[package]]
name = "polars-time" name = "polars-time"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c0d11ef8a93b2d204a60da7099510c40fe1ac04d95022ae4daa39df31b345e1" checksum = "53f42d2632f5971c9575041d33cbcfb1f996900c40bbf58bc6eb0a0c5efbecea"
dependencies = [ dependencies = [
"arrow2", "arrow2",
"atoi", "atoi",
@ -3887,16 +3871,19 @@ dependencies = [
[[package]] [[package]]
name = "polars-utils" name = "polars-utils"
version = "0.30.0" version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "145a59f928f8317fcf543407ef1b83e827448462e05e2fc6444162fcec84386a" checksum = "c326708a370d71dc6e11a8f4bbc10a8479e1c314dc048ba73543b815cd0bf339"
dependencies = [ dependencies = [
"ahash", "ahash",
"hashbrown 0.13.2", "hashbrown 0.14.0",
"num-traits",
"once_cell", "once_cell",
"polars-error",
"rayon", "rayon",
"smartstring", "smartstring",
"sysinfo", "sysinfo",
"version_check",
] ]
[[package]] [[package]]
@ -4631,7 +4618,7 @@ dependencies = [
"futures", "futures",
"lazy_static", "lazy_static",
"log", "log",
"parking_lot 0.12.1", "parking_lot",
"serial_test_derive", "serial_test_derive",
] ]
@ -4824,12 +4811,11 @@ dependencies = [
[[package]] [[package]]
name = "sqlparser" name = "sqlparser"
version = "0.34.0" version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37d3706eefb17039056234df6b566b0014f303f867f2656108334a55b8096f59" checksum = "2eaa1e88e78d2c2460d78b7dc3f0c08dbb606ab4222f9aff36f420d36e307d87"
dependencies = [ dependencies = [
"log", "log",
"serde",
] ]
[[package]] [[package]]
@ -4873,7 +4859,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"
dependencies = [ dependencies = [
"new_debug_unreachable", "new_debug_unreachable",
"once_cell", "once_cell",
"parking_lot 0.12.1", "parking_lot",
"phf_shared 0.10.0", "phf_shared 0.10.0",
"precomputed-hash", "precomputed-hash",
"serde", "serde",
@ -5238,7 +5224,7 @@ dependencies = [
"libc", "libc",
"mio", "mio",
"num_cpus", "num_cpus",
"parking_lot 0.12.1", "parking_lot",
"pin-project-lite", "pin-project-lite",
"signal-hook-registry", "signal-hook-registry",
"socket2", "socket2",
@ -5627,18 +5613,6 @@ dependencies = [
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03"
dependencies = [
"cfg-if",
"js-sys",
"wasm-bindgen",
"web-sys",
]
[[package]] [[package]]
name = "wasm-bindgen-macro" name = "wasm-bindgen-macro"
version = "0.2.87" version = "0.2.87"
@ -5668,21 +5642,6 @@ version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
[[package]]
name = "wasm-timer"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f"
dependencies = [
"futures",
"js-sys",
"parking_lot 0.11.2",
"pin-utils",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]] [[package]]
name = "wax" name = "wax"
version = "0.5.0" version = "0.5.0"

View file

@ -23,8 +23,8 @@ fancy-regex = "0.11"
indexmap = { version = "2.0" } indexmap = { version = "2.0" }
num = { version = "0.4", optional = true } num = { version = "0.4", optional = true }
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
sqlparser = { version = "0.34", features = ["serde"], optional = true } sqlparser = { version = "0.36.1", optional = true }
polars-io = { version = "0.30.0", features = ["avro"] } polars-io = { version = "0.32", features = ["avro"] }
[dependencies.polars] [dependencies.polars]
features = [ features = [
@ -54,7 +54,7 @@ features = [
"to_dummies", "to_dummies",
] ]
optional = true optional = true
version = "0.30.0" version = "0.32"
[features] [features]
dataframe = ["num", "polars", "sqlparser"] dataframe = ["num", "polars", "sqlparser"]

View file

@ -20,6 +20,7 @@ impl Command for Dummies {
fn signature(&self) -> Signature { fn signature(&self) -> Signature {
Signature::build(self.name()) Signature::build(self.name())
.switch("drop-first", "Drop first row", Some('d'))
.input_output_type( .input_output_type(
Type::Custom("dataframe".into()), Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()), Type::Custom("dataframe".into()),
@ -115,10 +116,11 @@ fn command(
call: &Call, call: &Call,
input: PipelineData, input: PipelineData,
) -> Result<PipelineData, ShellError> { ) -> Result<PipelineData, ShellError> {
let drop_first: bool = call.has_flag("drop-first");
let df = NuDataFrame::try_from_pipeline(input, call.head)?; let df = NuDataFrame::try_from_pipeline(input, call.head)?;
df.as_ref() df.as_ref()
.to_dummies(None) .to_dummies(None, drop_first)
.map_err(|e| { .map_err(|e| {
ShellError::GenericError( ShellError::GenericError(
"Error calculating dummies".into(), "Error calculating dummies".into(),

View file

@ -128,7 +128,7 @@ fn dataframe_command(
} }
fn expression_command(call: &Call, input: Value) -> Result<PipelineData, ShellError> { fn expression_command(call: &Call, input: Value) -> Result<PipelineData, ShellError> {
let expr = NuExpression::try_from_value(input)?; let expr = NuExpression::try_from_value(input)?;
let value = expr.to_value(call.head); let value = expr.to_value(call.head)?;
Ok(PipelineData::Value(value, None)) Ok(PipelineData::Value(value, None))
} }

View file

@ -95,10 +95,8 @@ impl Command for ExprOtherwise {
let value = input.into_value(call.head); let value = input.into_value(call.head);
let complete: NuExpression = match NuWhen::try_from_value(value)? { let complete: NuExpression = match NuWhen::try_from_value(value)? {
NuWhen::WhenThen(when_then) => when_then NuWhen::Then(then) => then.otherwise(otherwise_predicate.into_polars()).into(),
.otherwise(otherwise_predicate.into_polars()) NuWhen::ChainedThen(chained_when) => chained_when
.into(),
NuWhen::WhenThenThen(when_then_then) => when_then_then
.otherwise(otherwise_predicate.into_polars()) .otherwise(otherwise_predicate.into_polars())
.into(), .into(),
}; };

View file

@ -110,11 +110,11 @@ impl Command for ExprWhen {
.then(then_predicate.into_polars()) .then(then_predicate.into_polars())
.into(), .into(),
v => match NuWhen::try_from_value(v)? { v => match NuWhen::try_from_value(v)? {
NuWhen::WhenThen(when_then) => when_then NuWhen::Then(when_then) => when_then
.when(when_predicate.into_polars()) .when(when_predicate.into_polars())
.then(then_predicate.into_polars()) .then(then_predicate.into_polars())
.into(), .into(),
NuWhen::WhenThenThen(when_then_then) => when_then_then NuWhen::ChainedThen(when_then_then) => when_then_then
.when(when_predicate.into_polars()) .when(when_predicate.into_polars())
.then(then_predicate.into_polars()) .then(then_predicate.into_polars())
.into(), .into(),

View file

@ -172,7 +172,6 @@ fn get_col_name(expr: &Expr) -> Option<String> {
}, },
Expr::Filter { input: expr, .. } Expr::Filter { input: expr, .. }
| Expr::Slice { input: expr, .. } | Expr::Slice { input: expr, .. }
| Expr::Cache { input: expr, .. }
| Expr::Cast { expr, .. } | Expr::Cast { expr, .. }
| Expr::Sort { expr, .. } | Expr::Sort { expr, .. }
| Expr::Take { expr, .. } | Expr::Take { expr, .. }
@ -192,7 +191,8 @@ fn get_col_name(expr: &Expr) -> Option<String> {
| Expr::Wildcard | Expr::Wildcard
| Expr::RenameAlias { .. } | Expr::RenameAlias { .. }
| Expr::Count | Expr::Count
| Expr::Nth(_) => None, | Expr::Nth(_)
| Expr::Selector(_) => None,
} }
} }

View file

@ -37,6 +37,7 @@ impl Command for LazySortBy {
"nulls are shown last in the dataframe", "nulls are shown last in the dataframe",
Some('n'), Some('n'),
) )
.switch("maintain-order", "Maintains order during sort", Some('m'))
.input_output_type( .input_output_type(
Type::Custom("dataframe".into()), Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()), Type::Custom("dataframe".into()),
@ -110,6 +111,7 @@ impl Command for LazySortBy {
}; };
let expressions = NuExpression::extract_exprs(value)?; let expressions = NuExpression::extract_exprs(value)?;
let nulls_last = call.has_flag("nulls-last"); let nulls_last = call.has_flag("nulls-last");
let maintain_order = call.has_flag("maintain-order");
let reverse: Option<Vec<bool>> = call.get_flag(engine_state, stack, "reverse")?; let reverse: Option<Vec<bool>> = call.get_flag(engine_state, stack, "reverse")?;
let reverse = match reverse { let reverse = match reverse {
@ -137,7 +139,7 @@ impl Command for LazySortBy {
let lazy = NuLazyFrame::new( let lazy = NuLazyFrame::new(
lazy.from_eager, lazy.from_eager,
lazy.into_polars() lazy.into_polars()
.sort_by_exprs(&expressions, reverse, nulls_last), .sort_by_exprs(&expressions, reverse, nulls_last, maintain_order),
); );
Ok(PipelineData::Value( Ok(PipelineData::Value(

View file

@ -143,7 +143,13 @@ fn command(
})?; })?;
let res = if not_exact { let res = if not_exact {
casted.as_datetime_not_exact(Some(format.as_str()), TimeUnit::Nanoseconds, None) casted.as_datetime_not_exact(
Some(format.as_str()),
TimeUnit::Nanoseconds,
false,
None,
None,
)
} else { } else {
casted.as_datetime( casted.as_datetime(
Some(format.as_str()), Some(format.as_str()),
@ -151,6 +157,7 @@ fn command(
false, false,
false, false,
None, None,
None,
) )
}; };

View file

@ -27,6 +27,11 @@ impl Command for ArgSort {
Signature::build(self.name()) Signature::build(self.name())
.switch("reverse", "reverse order", Some('r')) .switch("reverse", "reverse order", Some('r'))
.switch("nulls-last", "nulls ordered last", Some('n')) .switch("nulls-last", "nulls ordered last", Some('n'))
.switch(
"maintain-order",
"maintain order on sorted items",
Some('m'),
)
.input_output_type( .input_output_type(
Type::Custom("dataframe".into()), Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()), Type::Custom("dataframe".into()),
@ -98,6 +103,7 @@ fn command(
descending: call.has_flag("reverse"), descending: call.has_flag("reverse"),
nulls_last: call.has_flag("nulls-last"), nulls_last: call.has_flag("nulls-last"),
multithreaded: true, multithreaded: true,
maintain_order: call.has_flag("maintain-order"),
}; };
let mut res = df let mut res = df

View file

@ -98,7 +98,12 @@ fn command(
let value = NuDataFrame::dataframe_into_value(res, call.head); let value = NuDataFrame::dataframe_into_value(res, call.head);
Ok(PipelineData::Value(value, None)) Ok(PipelineData::Value(value, None))
} }
_ => todo!(), _ => Err(ShellError::UnsupportedInput(
"Expected the dataframe to have a column".to_string(),
"".to_string(),
call.head,
call.head,
)),
} }
} }

View file

@ -149,6 +149,7 @@ fn command(
closed_window: None, closed_window: None,
tu: None, tu: None,
tz: None, tz: None,
fn_params: None,
}; };
let res = match roll_type { let res = match roll_type {
RollType::Max => series.rolling_max(rolling_opts), RollType::Max => series.rolling_max(rolling_opts),

View file

@ -421,7 +421,17 @@ fn input_type_list_to_series(
let dt_chunked = ChunkedArray::<Int64Type>::from_iter_options(&list_name, it) let dt_chunked = ChunkedArray::<Int64Type>::from_iter_options(&list_name, it)
.into_datetime(TimeUnit::Nanoseconds, None); .into_datetime(TimeUnit::Nanoseconds, None);
builder.append_series(&dt_chunked.into_series()); builder
.append_series(&dt_chunked.into_series())
.map_err(|e| {
ShellError::GenericError(
"Error appending to series".into(),
"".to_string(),
None,
Some(e.to_string()),
Vec::new(),
)
})?
} }
let res = builder.finish(); let res = builder.finish();
Ok(res.into_series()) Ok(res.into_series())

View file

@ -466,12 +466,12 @@ impl NuDataFrame {
.expect("already checked that dataframe is different than 0"); .expect("already checked that dataframe is different than 0");
// if unable to sort, then unable to compare // if unable to sort, then unable to compare
let lhs = match self.as_ref().sort(vec![*first_col], false) { let lhs = match self.as_ref().sort(vec![*first_col], false, false) {
Ok(df) => df, Ok(df) => df,
Err(_) => return None, Err(_) => return None,
}; };
let rhs = match other.as_ref().sort(vec![*first_col], false) { let rhs = match other.as_ref().sort(vec![*first_col], false, false) {
Ok(df) => df, Ok(df) => df,
Err(_) => return None, Err(_) => return None,
}; };

View file

@ -31,7 +31,7 @@ impl CustomValue for NuExpression {
} }
fn to_base_value(&self, span: Span) -> Result<Value, ShellError> { fn to_base_value(&self, span: Span) -> Result<Value, ShellError> {
Ok(self.to_value(span)) self.to_value(span)
} }
fn as_any(&self) -> &dyn std::any::Any { fn as_any(&self) -> &dyn std::any::Any {

View file

@ -115,7 +115,7 @@ impl NuExpression {
f(expr, other).into() f(expr, other).into()
} }
pub fn to_value(&self, span: Span) -> Value { pub fn to_value(&self, span: Span) -> Result<Value, ShellError> {
expr_to_value(self.as_ref(), span) expr_to_value(self.as_ref(), span)
} }
@ -164,60 +164,59 @@ impl ExtractedExpr {
} }
} }
pub fn expr_to_value(expr: &Expr, span: Span) -> Value { pub fn expr_to_value(expr: &Expr, span: Span) -> Result<Value, ShellError> {
match expr { match expr {
Expr::Alias(expr, alias) => Value::record( Expr::Alias(expr, alias) => Ok(Value::record(
record! { record! {
"expr" => expr_to_value(expr.as_ref(), span), "expr" => expr_to_value(expr.as_ref(), span)?,
"alias" => Value::string(alias.as_ref(), span), "alias" => Value::string(alias.as_ref(), span),
}, },
span, span,
), )),
Expr::Column(name) => Ok(Value::record(
Expr::Column(name) => Value::record(
record! { record! {
"expr" => Value::string("column", span), "expr" => Value::string("column", span),
"value" => Value::string(name.to_string(), span), "value" => Value::string(name.to_string(), span),
}, },
span, span,
), )),
Expr::Columns(columns) => { Expr::Columns(columns) => {
let value = columns.iter().map(|col| Value::string(col, span)).collect(); let value = columns.iter().map(|col| Value::string(col, span)).collect();
Value::record( Ok(Value::record(
record! { record! {
"expr" => Value::string("columns", span), "expr" => Value::string("columns", span),
"value" => Value::list(value, span), "value" => Value::list(value, span),
}, },
span, span,
) ))
} }
Expr::Literal(literal) => Value::record( Expr::Literal(literal) => Ok(Value::record(
record! { record! {
"expr" => Value::string("literal", span), "expr" => Value::string("literal", span),
"value" => Value::string(format!("{literal:?}"), span), "value" => Value::string(format!("{literal:?}"), span),
}, },
span, span,
), )),
Expr::BinaryExpr { left, op, right } => Value::record( Expr::BinaryExpr { left, op, right } => Ok(Value::record(
record! { record! {
"left" => expr_to_value(left, span), "left" => expr_to_value(left, span)?,
"op" => Value::string(format!("{op:?}"), span), "op" => Value::string(format!("{op:?}"), span),
"right" => expr_to_value(right, span), "right" => expr_to_value(right, span)?,
}, },
span, span,
), )),
Expr::Ternary { Expr::Ternary {
predicate, predicate,
truthy, truthy,
falsy, falsy,
} => Value::record( } => Ok(Value::record(
record! { record! {
"predicate" => expr_to_value(predicate.as_ref(), span), "predicate" => expr_to_value(predicate.as_ref(), span)?,
"truthy" => expr_to_value(truthy.as_ref(), span), "truthy" => expr_to_value(truthy.as_ref(), span)?,
"falsy" => expr_to_value(falsy.as_ref(), span), "falsy" => expr_to_value(falsy.as_ref(), span)?,
}, },
span, span,
), )),
Expr::Agg(agg_expr) => { Expr::Agg(agg_expr) => {
let value = match agg_expr { let value = match agg_expr {
AggExpr::Min { input: expr, .. } AggExpr::Min { input: expr, .. }
@ -237,37 +236,44 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value {
expr, expr,
quantile, quantile,
interpol, interpol,
} => Value::record( } => Ok(Value::record(
record! { record! {
"expr" => expr_to_value(expr.as_ref(), span), "expr" => expr_to_value(expr.as_ref(), span)?,
"quantile" => expr_to_value(quantile.as_ref(), span), "quantile" => expr_to_value(quantile.as_ref(), span)?,
"interpol" => Value::string(format!("{interpol:?}"), span), "interpol" => Value::string(format!("{interpol:?}"), span),
}, },
span, span,
), )),
}; };
Value::record( Ok(Value::record(
record! { record! {
"expr" => Value::string("agg", span), "expr" => Value::string("agg", span),
"value" => value, "value" => value?,
}, },
span, span,
) ))
} }
Expr::Count => Value::record(record! { "expr" => Value::string("count", span) }, span), Expr::Count => Ok(Value::record(
Expr::Wildcard => { record! { "expr" => Value::string("count", span) },
Value::record(record! { "expr" => Value::string("wildcard", span) }, span)
}
Expr::Explode(expr) => Value::record(
record! { "expr" => expr_to_value(expr.as_ref(), span) },
span, span,
), )),
Expr::KeepName(expr) => Value::record( Expr::Wildcard => Ok(Value::record(
record! { "expr" => expr_to_value(expr.as_ref(), span) }, record! { "expr" => Value::string("wildcard", span) },
span, span,
), )),
Expr::Nth(i) => Value::record(record! { "expr" => Value::int(*i, span) }, span), Expr::Explode(expr) => Ok(Value::record(
record! { "expr" => expr_to_value(expr.as_ref(), span)? },
span,
)),
Expr::KeepName(expr) => Ok(Value::record(
record! { "expr" => expr_to_value(expr.as_ref(), span)? },
span,
)),
Expr::Nth(i) => Ok(Value::record(
record! { "expr" => Value::int(*i, span) },
span,
)),
Expr::DtypeColumn(dtypes) => { Expr::DtypeColumn(dtypes) => {
let vals = dtypes let vals = dtypes
.iter() .iter()
@ -277,73 +283,74 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value {
}) })
.collect(); .collect();
Value::List { vals, span } Ok(Value::List { vals, span })
} }
Expr::Sort { expr, options } => Value::record( Expr::Sort { expr, options } => Ok(Value::record(
record! { record! {
"expr" => expr_to_value(expr.as_ref(), span), "expr" => expr_to_value(expr.as_ref(), span)?,
"options" => Value::string(format!("{options:?}"), span), "options" => Value::string(format!("{options:?}"), span),
}, },
span, span,
), )),
Expr::Cast { Expr::Cast {
expr, expr,
data_type, data_type,
strict, strict,
} => Value::record( } => Ok(Value::record(
record! { record! {
"expr" => expr_to_value(expr.as_ref(), span), "expr" => expr_to_value(expr.as_ref(), span)?,
"dtype" => Value::string(format!("{data_type:?}"), span), "dtype" => Value::string(format!("{data_type:?}"), span),
"strict" => Value::bool(*strict, span), "strict" => Value::bool(*strict, span),
}, },
span, span,
), )),
Expr::Take { expr, idx } => Value::record( Expr::Take { expr, idx } => Ok(Value::record(
record! { record! {
"expr" => expr_to_value(expr.as_ref(), span), "expr" => expr_to_value(expr.as_ref(), span)?,
"idx" => expr_to_value(idx.as_ref(), span), "idx" => expr_to_value(idx.as_ref(), span)?,
}, },
span, span,
), )),
Expr::SortBy { Expr::SortBy {
expr, expr,
by, by,
descending, descending,
} => { } => {
let by: Vec<Value> = by.iter().map(|b| expr_to_value(b, span)).collect(); let by: Result<Vec<Value>, ShellError> =
by.iter().map(|b| expr_to_value(b, span)).collect();
let descending: Vec<Value> = descending let descending: Vec<Value> = descending
.iter() .iter()
.map(|r| Value::Bool { val: *r, span }) .map(|r| Value::Bool { val: *r, span })
.collect(); .collect();
Value::record( Ok(Value::record(
record! { record! {
"expr" => expr_to_value(expr.as_ref(), span), "expr" => expr_to_value(expr.as_ref(), span)?,
"by" => Value::list(by, span), "by" => Value::list(by?, span),
"descending" => Value::list(descending, span), "descending" => Value::list(descending, span),
}, },
span, span,
) ))
} }
Expr::Filter { input, by } => Value::record( Expr::Filter { input, by } => Ok(Value::record(
record! { record! {
"input" => expr_to_value(input.as_ref(), span), "input" => expr_to_value(input.as_ref(), span)?,
"by" => expr_to_value(by.as_ref(), span), "by" => expr_to_value(by.as_ref(), span)?,
}, },
span, span,
), )),
Expr::Slice { Expr::Slice {
input, input,
offset, offset,
length, length,
} => Value::record( } => Ok(Value::record(
record! { record! {
"input" => expr_to_value(input.as_ref(), span), "input" => expr_to_value(input.as_ref(), span)?,
"offset" => expr_to_value(offset.as_ref(), span), "offset" => expr_to_value(offset.as_ref(), span)?,
"length" => expr_to_value(length.as_ref(), span), "length" => expr_to_value(length.as_ref(), span)?,
}, },
span, span,
), )),
Expr::Exclude(expr, excluded) => { Expr::Exclude(expr, excluded) => {
let excluded = excluded let excluded = excluded
.iter() .iter()
@ -353,67 +360,62 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value {
}) })
.collect(); .collect();
Value::record( Ok(Value::record(
record! { record! {
"expr" => expr_to_value(expr.as_ref(), span), "expr" => expr_to_value(expr.as_ref(), span)?,
"excluded" => Value::list(excluded, span), "excluded" => Value::list(excluded, span),
}, },
span, span,
) ))
} }
Expr::RenameAlias { expr, function } => Value::record( Expr::RenameAlias { expr, function } => Ok(Value::record(
record! { record! {
"expr" => expr_to_value(expr.as_ref(), span), "expr" => expr_to_value(expr.as_ref(), span)?,
"function" => Value::string(format!("{function:?}"), span), "function" => Value::string(format!("{function:?}"), span),
}, },
span, span,
), )),
Expr::AnonymousFunction { Expr::AnonymousFunction {
input, input,
function, function,
output_type, output_type,
options, options,
} => { } => {
let input: Vec<Value> = input.iter().map(|e| expr_to_value(e, span)).collect(); let input: Result<Vec<Value>, ShellError> =
Value::record( input.iter().map(|e| expr_to_value(e, span)).collect();
Ok(Value::record(
record! { record! {
"input" => Value::list(input, span), "input" => Value::list(input?, span),
"function" => Value::string(format!("{function:?}"), span), "function" => Value::string(format!("{function:?}"), span),
"output_type" => Value::string(format!("{output_type:?}"), span), "output_type" => Value::string(format!("{output_type:?}"), span),
"options" => Value::string(format!("{options:?}"), span), "options" => Value::string(format!("{options:?}"), span),
}, },
span, span,
) ))
} }
Expr::Function { Expr::Function {
input, input,
function, function,
options, options,
} => { } => {
let input: Vec<Value> = input.iter().map(|e| expr_to_value(e, span)).collect(); let input: Result<Vec<Value>, ShellError> =
Value::record( input.iter().map(|e| expr_to_value(e, span)).collect();
Ok(Value::record(
record! { record! {
"input" => Value::list(input, span), "input" => Value::list(input?, span),
"function" => Value::string(format!("{function:?}"), span), "function" => Value::string(format!("{function:?}"), span),
"options" => Value::string(format!("{options:?}"), span), "options" => Value::string(format!("{options:?}"), span),
}, },
span, span,
) ))
} }
Expr::Cache { input, id } => Value::record(
record! {
"input" => expr_to_value(input.as_ref(), span),
"id" => Value::string(format!("{id:?}"), span),
},
span,
),
Expr::Window { Expr::Window {
function, function,
partition_by, partition_by,
order_by, order_by,
options, options,
} => { } => {
let partition_by: Vec<Value> = partition_by let partition_by: Result<Vec<Value>, ShellError> = partition_by
.iter() .iter()
.map(|e| expr_to_value(e, span)) .map(|e| expr_to_value(e, span))
.collect(); .collect();
@ -421,17 +423,26 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value {
let order_by = order_by let order_by = order_by
.as_ref() .as_ref()
.map(|e| expr_to_value(e.as_ref(), span)) .map(|e| expr_to_value(e.as_ref(), span))
.transpose()?
.unwrap_or_else(|| Value::nothing(span)); .unwrap_or_else(|| Value::nothing(span));
Value::record( Ok(Value::record(
record! { record! {
"function" => expr_to_value(function, span), "function" => expr_to_value(function, span)?,
"partition_by" => Value::list(partition_by, span), "partition_by" => Value::list(partition_by?, span),
"order_by" => order_by, "order_by" => order_by,
"options" => Value::string(format!("{options:?}"), span), "options" => Value::string(format!("{options:?}"), span),
}, },
span, span,
) ))
} }
// the parameter polars_plan::dsl::selector::Selector is not publicly exposed.
// I am not sure what we can meaningfully do with this at this time.
Expr::Selector(_) => Err(ShellError::UnsupportedInput(
"Expressions of type Selector to Nu Values is not yet supported".to_string(),
format!("Expression is {expr:?}"),
span,
Span::unknown(),
)),
} }
} }

View file

@ -26,8 +26,8 @@ impl CustomValue for NuWhen {
fn to_base_value(&self, span: Span) -> Result<Value, ShellError> { fn to_base_value(&self, span: Span) -> Result<Value, ShellError> {
let val = match self { let val = match self {
NuWhen::WhenThen(_) => "whenthen".into(), NuWhen::Then(_) => "whenthen".into(),
NuWhen::WhenThenThen(_) => "whenthenthen".into(), NuWhen::ChainedThen(_) => "whenthenthen".into(),
}; };
let value = Value::String { val, span }; let value = Value::String { val, span };

View file

@ -2,13 +2,13 @@ mod custom_value;
use core::fmt; use core::fmt;
use nu_protocol::{ShellError, Span, Value}; use nu_protocol::{ShellError, Span, Value};
use polars::prelude::{col, when, WhenThen, WhenThenThen}; use polars::prelude::{col, when, ChainedThen, Then};
use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Clone)] #[derive(Clone)]
pub enum NuWhen { pub enum NuWhen {
WhenThen(Box<WhenThen>), Then(Box<Then>),
WhenThenThen(WhenThenThen), ChainedThen(ChainedThen),
} }
// Mocked serialization of the LazyFrame object // Mocked serialization of the LazyFrame object
@ -27,7 +27,7 @@ impl<'de> Deserialize<'de> for NuWhen {
where where
D: Deserializer<'de>, D: Deserializer<'de>,
{ {
Ok(NuWhen::WhenThen(Box::new(when(col("a")).then(col("b"))))) Ok(NuWhen::Then(Box::new(when(col("a")).then(col("b")))))
} }
} }
@ -37,15 +37,15 @@ impl fmt::Debug for NuWhen {
} }
} }
impl From<WhenThen> for NuWhen { impl From<Then> for NuWhen {
fn from(when_then: WhenThen) -> Self { fn from(then: Then) -> Self {
NuWhen::WhenThen(Box::new(when_then)) NuWhen::Then(Box::new(then))
} }
} }
impl From<WhenThenThen> for NuWhen { impl From<ChainedThen> for NuWhen {
fn from(when_then_then: WhenThenThen) -> Self { fn from(chained_when: ChainedThen) -> Self {
NuWhen::WhenThenThen(when_then_then) NuWhen::ChainedThen(chained_when)
} }
} }

View file

@ -80,7 +80,6 @@ serde_json = "1.0"
serde_urlencoded = "0.7" serde_urlencoded = "0.7"
serde_yaml = "0.9" serde_yaml = "0.9"
sha2 = "0.10" sha2 = "0.10"
sqlparser = { version = "0.34", features = ["serde"], optional = true }
sysinfo = "0.29" sysinfo = "0.29"
tabled = { version = "0.14.0", features = ["color"], default-features = false } tabled = { version = "0.14.0", features = ["color"], default-features = false }
terminal_size = "0.2" terminal_size = "0.2"