Mirrors/nushell
2
0
Fork 0
mirror of https://github.com/nushell/nushell synced 2024-12-27 05:23:11 +00:00
Code Issues Projects Releases Packages Wiki Activity

Upgrading nu-cmd-dataframe to polars 0.39 (#12554)

Browse source 
#Description
Upgrading nu-cmd-dataframe to polars 0.39

---------

Co-authored-by: Jack Wright <jack.wright@disqo.com>
This commit is contained in:
Jack Wright 2024-04-17 10:50:17 -07:00 committed by GitHub
parent 13160b3ec3
commit 57b0c722c6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 127 additions and 497 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

571
Cargo.lock generated
View file

@ -2953,12 +2953,12 @@ dependencies = [
"nu-parser", "nu-parser",
"nu-protocol", "nu-protocol",
"num", "num",
"polars 0.38.3", "polars",
"polars-arrow 0.38.3", "polars-arrow",
"polars-io 0.38.3", "polars-io",
"polars-ops 0.38.3", "polars-ops",
"polars-plan 0.38.3", "polars-plan",
"polars-utils 0.38.3", "polars-utils",
"serde", "serde",
"sqlparser 0.45.0", "sqlparser 0.45.0",
] ]
@ -3428,12 +3428,12 @@ dependencies = [
"nu-plugin-test-support", "nu-plugin-test-support",
"nu-protocol", "nu-protocol",
"num", "num",
"polars 0.39.1", "polars",
"polars-arrow 0.39.1", "polars-arrow",
"polars-io 0.39.1", "polars-io",
"polars-ops 0.39.1", "polars-ops",
"polars-plan 0.39.1", "polars-plan",
"polars-utils 0.39.1", "polars-utils",
"serde", "serde",
"sqlparser 0.45.0", "sqlparser 0.45.0",
"tempfile", "tempfile",
@ -4007,26 +4007,6 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "polars"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f01006048a264047d6cba081fed8e11adbd69c15956f9e53185a9ac4a541853c"
dependencies = [
"getrandom",
"polars-arrow 0.38.3",
"polars-core 0.38.3",
"polars-error 0.38.3",
"polars-io 0.38.3",
"polars-lazy 0.38.3",
"polars-ops 0.38.3",
"polars-parquet 0.38.3",
"polars-sql 0.38.3",
"polars-time 0.38.3",
"polars-utils 0.38.3",
"version_check",
]
[[package]] [[package]]
name = "polars" name = "polars"
version = "0.39.1" version = "0.39.1"
@ -4034,57 +4014,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c352aaa0399c0863eecd879f2cbe585c9026c5cafe432f029025e4bec3adf43" checksum = "1c352aaa0399c0863eecd879f2cbe585c9026c5cafe432f029025e4bec3adf43"
dependencies = [ dependencies = [
"getrandom", "getrandom",
"polars-arrow 0.39.1", "polars-arrow",
"polars-core 0.39.1", "polars-core",
"polars-error 0.39.1", "polars-error",
"polars-io 0.39.1", "polars-io",
"polars-lazy 0.39.1", "polars-lazy",
"polars-ops 0.39.1", "polars-ops",
"polars-parquet 0.39.1", "polars-parquet",
"polars-sql 0.39.1", "polars-sql",
"polars-time 0.39.1", "polars-time",
"polars-utils 0.39.1", "polars-utils",
"version_check", "version_check",
] ]
[[package]]
name = "polars-arrow"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25197f40d71f82b2f79bb394f03e555d3cc1ce4db1dd052c28318721c71e96ad"
dependencies = [
"ahash 0.8.11",
"atoi",
"atoi_simd",
"avro-schema",
"bytemuck",
"chrono",
"chrono-tz 0.8.6",
"dyn-clone",
"either",
"ethnum",
"fast-float",
"foreign_vec",
"futures",
"getrandom",
"hashbrown 0.14.3",
"itoa",
"itoap",
"lz4",
"multiversion",
"num-traits",
"polars-arrow-format",
"polars-error 0.38.3",
"polars-utils 0.38.3",
"ryu",
"serde",
"simdutf8",
"streaming-iterator",
"strength_reduce",
"version_check",
"zstd",
]
[[package]] [[package]]
name = "polars-arrow" name = "polars-arrow"
version = "0.39.1" version = "0.39.1"
@ -4112,8 +4054,8 @@ dependencies = [
"multiversion", "multiversion",
"num-traits", "num-traits",
"polars-arrow-format", "polars-arrow-format",
"polars-error 0.39.1", "polars-error",
"polars-utils 0.39.1", "polars-utils",
"ryu", "ryu",
"serde", "serde",
"simdutf8", "simdutf8",
@ -4133,22 +4075,6 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "polars-compute"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c354515f73cdbbad03c2bf723fcd68e6825943b3ec503055abc8a8cb08ce46bb"
dependencies = [
"bytemuck",
"either",
"num-traits",
"polars-arrow 0.38.3",
"polars-error 0.38.3",
"polars-utils 0.38.3",
"strength_reduce",
"version_check",
]
[[package]] [[package]]
name = "polars-compute" name = "polars-compute"
version = "0.39.1" version = "0.39.1"
@ -4158,46 +4084,13 @@ dependencies = [
"bytemuck", "bytemuck",
"either", "either",
"num-traits", "num-traits",
"polars-arrow 0.39.1", "polars-arrow",
"polars-error 0.39.1", "polars-error",
"polars-utils 0.39.1", "polars-utils",
"strength_reduce", "strength_reduce",
"version_check", "version_check",
] ]
[[package]]
name = "polars-core"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f20d3c227186f74aa3c228c64ef72f5a15617322fed30b4323eaf53b25f8e7b"
dependencies = [
"ahash 0.8.11",
"bitflags 2.5.0",
"bytemuck",
"chrono",
"chrono-tz 0.8.6",
"either",
"hashbrown 0.14.3",
"indexmap",
"num-traits",
"once_cell",
"polars-arrow 0.38.3",
"polars-compute 0.38.3",
"polars-error 0.38.3",
"polars-row 0.38.3",
"polars-utils 0.38.3",
"rand",
"rand_distr",
"rayon",
"regex",
"serde",
"serde_json",
"smartstring",
"thiserror",
"version_check",
"xxhash-rust",
]
[[package]] [[package]]
name = "polars-core" name = "polars-core"
version = "0.39.1" version = "0.39.1"
@ -4215,11 +4108,11 @@ dependencies = [
"indexmap", "indexmap",
"num-traits", "num-traits",
"once_cell", "once_cell",
"polars-arrow 0.39.1", "polars-arrow",
"polars-compute 0.39.1", "polars-compute",
"polars-error 0.39.1", "polars-error",
"polars-row 0.39.1", "polars-row",
"polars-utils 0.39.1", "polars-utils",
"rand", "rand",
"rand_distr", "rand_distr",
"rayon", "rayon",
@ -4232,19 +4125,6 @@ dependencies = [
"xxhash-rust", "xxhash-rust",
] ]
[[package]]
name = "polars-error"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d66dd0ce51f8bd620eb8bd376502fe68a2b1a446d5433ecd2e75270b0755ce76"
dependencies = [
"avro-schema",
"polars-arrow-format",
"regex",
"simdutf8",
"thiserror",
]
[[package]] [[package]]
name = "polars-error" name = "polars-error"
version = "0.39.1" version = "0.39.1"
@ -4258,47 +4138,6 @@ dependencies = [
"thiserror", "thiserror",
] ]
[[package]]
name = "polars-io"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b40bef2edcdc58394792c4d779465144283a09ff1836324e7b72df7978a6e992"
dependencies = [
"ahash 0.8.11",
"async-trait",
"atoi_simd",
"bytes",
"chrono",
"fast-float",
"flate2",
"futures",
"home",
"itoa",
"memchr",
"memmap2",
"num-traits",
"once_cell",
"percent-encoding",
"polars-arrow 0.38.3",
"polars-core 0.38.3",
"polars-error 0.38.3",
"polars-json 0.38.3",
"polars-parquet 0.38.3",
"polars-time 0.38.3",
"polars-utils 0.38.3",
"rayon",
"regex",
"ryu",
"serde",
"serde_json",
"simd-json",
"simdutf8",
"smartstring",
"tokio",
"tokio-util",
"zstd",
]
[[package]] [[package]]
name = "polars-io" name = "polars-io"
version = "0.39.1" version = "0.39.1"
@ -4320,13 +4159,13 @@ dependencies = [
"num-traits", "num-traits",
"once_cell", "once_cell",
"percent-encoding", "percent-encoding",
"polars-arrow 0.39.1", "polars-arrow",
"polars-core 0.39.1", "polars-core",
"polars-error 0.39.1", "polars-error",
"polars-json 0.39.1", "polars-json",
"polars-parquet 0.39.1", "polars-parquet",
"polars-time 0.39.1", "polars-time",
"polars-utils 0.39.1", "polars-utils",
"rayon", "rayon",
"regex", "regex",
"ryu", "ryu",
@ -4340,27 +4179,6 @@ dependencies = [
"zstd", "zstd",
] ]
[[package]]
name = "polars-json"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef86aca08f10ddc939fe95aabb44e1d2582dcb08b55d4dadb93353ce42adc248"
dependencies = [
"ahash 0.8.11",
"chrono",
"fallible-streaming-iterator",
"hashbrown 0.14.3",
"indexmap",
"itoa",
"num-traits",
"polars-arrow 0.38.3",
"polars-error 0.38.3",
"polars-utils 0.38.3",
"ryu",
"simd-json",
"streaming-iterator",
]
[[package]] [[package]]
name = "polars-json" name = "polars-json"
version = "0.39.1" version = "0.39.1"
@ -4374,38 +4192,14 @@ dependencies = [
"indexmap", "indexmap",
"itoa", "itoa",
"num-traits", "num-traits",
"polars-arrow 0.39.1", "polars-arrow",
"polars-error 0.39.1", "polars-error",
"polars-utils 0.39.1", "polars-utils",
"ryu", "ryu",
"simd-json", "simd-json",
"streaming-iterator", "streaming-iterator",
] ]
[[package]]
name = "polars-lazy"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c27df26a19d3092298d31d47614ad84dc330c106e38aa8cd53727cd91c07cf56"
dependencies = [
"ahash 0.8.11",
"bitflags 2.5.0",
"glob",
"once_cell",
"polars-arrow 0.38.3",
"polars-core 0.38.3",
"polars-io 0.38.3",
"polars-json 0.38.3",
"polars-ops 0.38.3",
"polars-pipe 0.38.3",
"polars-plan 0.38.3",
"polars-time 0.38.3",
"polars-utils 0.38.3",
"rayon",
"smartstring",
"version_check",
]
[[package]] [[package]]
name = "polars-lazy" name = "polars-lazy"
version = "0.39.1" version = "0.39.1"
@ -4416,56 +4210,20 @@ dependencies = [
"bitflags 2.5.0", "bitflags 2.5.0",
"glob", "glob",
"once_cell", "once_cell",
"polars-arrow 0.39.1", "polars-arrow",
"polars-core 0.39.1", "polars-core",
"polars-io 0.39.1", "polars-io",
"polars-json 0.39.1", "polars-json",
"polars-ops 0.39.1", "polars-ops",
"polars-pipe 0.39.1", "polars-pipe",
"polars-plan 0.39.1", "polars-plan",
"polars-time 0.39.1", "polars-time",
"polars-utils 0.39.1", "polars-utils",
"rayon", "rayon",
"smartstring", "smartstring",
"version_check", "version_check",
] ]
[[package]]
name = "polars-ops"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f8a51c3bdc9e7c34196ff6f5c3cb17da134e5aafb1756aaf24b76c7118e63dc"
dependencies = [
"ahash 0.8.11",
"argminmax",
"base64 0.21.7",
"bytemuck",
"chrono",
"chrono-tz 0.8.6",
"either",
"hashbrown 0.14.3",
"hex",
"indexmap",
"jsonpath_lib_polars_vendor",
"memchr",
"num-traits",
"polars-arrow 0.38.3",
"polars-compute 0.38.3",
"polars-core 0.38.3",
"polars-error 0.38.3",
"polars-json 0.38.3",
"polars-utils 0.38.3",
"rand",
"rand_distr",
"rayon",
"regex",
"serde",
"serde_json",
"smartstring",
"unicode-reverse",
"version_check",
]
[[package]] [[package]]
name = "polars-ops" name = "polars-ops"
version = "0.39.1" version = "0.39.1"
@ -4485,12 +4243,12 @@ dependencies = [
"jsonpath_lib_polars_vendor", "jsonpath_lib_polars_vendor",
"memchr", "memchr",
"num-traits", "num-traits",
"polars-arrow 0.39.1", "polars-arrow",
"polars-compute 0.39.1", "polars-compute",
"polars-core 0.39.1", "polars-core",
"polars-error 0.39.1", "polars-error",
"polars-json 0.39.1", "polars-json",
"polars-utils 0.39.1", "polars-utils",
"rand", "rand",
"rand_distr", "rand_distr",
"rayon", "rayon",
@ -4502,32 +4260,6 @@ dependencies = [
"version_check", "version_check",
] ]
[[package]]
name = "polars-parquet"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8824ee00fbbe83d69553f2711014c50361238d210ed81a7a297695b7db97d42"
dependencies = [
"ahash 0.8.11",
"async-stream",
"base64 0.21.7",
"brotli",
"ethnum",
"flate2",
"futures",
"lz4",
"num-traits",
"parquet-format-safe",
"polars-arrow 0.38.3",
"polars-error 0.38.3",
"polars-utils 0.38.3",
"seq-macro",
"simdutf8",
"snap",
"streaming-decompression",
"zstd",
]
[[package]] [[package]]
name = "polars-parquet" name = "polars-parquet"
version = "0.39.1" version = "0.39.1"
@ -4544,9 +4276,9 @@ dependencies = [
"lz4", "lz4",
"num-traits", "num-traits",
"parquet-format-safe", "parquet-format-safe",
"polars-arrow 0.39.1", "polars-arrow",
"polars-error 0.39.1", "polars-error",
"polars-utils 0.39.1", "polars-utils",
"seq-macro", "seq-macro",
"simdutf8", "simdutf8",
"snap", "snap",
@ -4554,31 +4286,6 @@ dependencies = [
"zstd", "zstd",
] ]
[[package]]
name = "polars-pipe"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c5e2c1f14e81d60cfa9afe4e611a9bad9631a2cb7cd19b7c0094d0dc32f0231"
dependencies = [
"crossbeam-channel",
"crossbeam-queue",
"enum_dispatch",
"hashbrown 0.14.3",
"num-traits",
"polars-arrow 0.38.3",
"polars-compute 0.38.3",
"polars-core 0.38.3",
"polars-io 0.38.3",
"polars-ops 0.38.3",
"polars-plan 0.38.3",
"polars-row 0.38.3",
"polars-utils 0.38.3",
"rayon",
"smartstring",
"uuid",
"version_check",
]
[[package]] [[package]]
name = "polars-pipe" name = "polars-pipe"
version = "0.39.1" version = "0.39.1"
@ -4590,47 +4297,20 @@ dependencies = [
"enum_dispatch", "enum_dispatch",
"hashbrown 0.14.3", "hashbrown 0.14.3",
"num-traits", "num-traits",
"polars-arrow 0.39.1", "polars-arrow",
"polars-compute 0.39.1", "polars-compute",
"polars-core 0.39.1", "polars-core",
"polars-io 0.39.1", "polars-io",
"polars-ops 0.39.1", "polars-ops",
"polars-plan 0.39.1", "polars-plan",
"polars-row 0.39.1", "polars-row",
"polars-utils 0.39.1", "polars-utils",
"rayon", "rayon",
"smartstring", "smartstring",
"uuid", "uuid",
"version_check", "version_check",
] ]
[[package]]
name = "polars-plan"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff48362bd1b078bbbec7e7ba9ec01fea58fee2887db22a8e3deaf78f322fa3c4"
dependencies = [
"ahash 0.8.11",
"bytemuck",
"chrono-tz 0.8.6",
"once_cell",
"percent-encoding",
"polars-arrow 0.38.3",
"polars-core 0.38.3",
"polars-io 0.38.3",
"polars-json 0.38.3",
"polars-ops 0.38.3",
"polars-parquet 0.38.3",
"polars-time 0.38.3",
"polars-utils 0.38.3",
"rayon",
"regex",
"serde",
"smartstring",
"strum_macros 0.25.3",
"version_check",
]
[[package]] [[package]]
name = "polars-plan" name = "polars-plan"
version = "0.39.1" version = "0.39.1"
@ -4643,14 +4323,14 @@ dependencies = [
"hashbrown 0.14.3", "hashbrown 0.14.3",
"once_cell", "once_cell",
"percent-encoding", "percent-encoding",
"polars-arrow 0.39.1", "polars-arrow",
"polars-core 0.39.1", "polars-core",
"polars-io 0.39.1", "polars-io",
"polars-json 0.39.1", "polars-json",
"polars-ops 0.39.1", "polars-ops",
"polars-parquet 0.39.1", "polars-parquet",
"polars-time 0.39.1", "polars-time",
"polars-utils 0.39.1", "polars-utils",
"rayon", "rayon",
"recursive", "recursive",
"regex", "regex",
@ -4660,18 +4340,6 @@ dependencies = [
"version_check", "version_check",
] ]
[[package]]
name = "polars-row"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63029da56ff6a720b190490bbc7b6263f9b72d1134311b1f381fc8d306d37770"
dependencies = [
"bytemuck",
"polars-arrow 0.38.3",
"polars-error 0.38.3",
"polars-utils 0.38.3",
]
[[package]] [[package]]
name = "polars-row" name = "polars-row"
version = "0.39.1" version = "0.39.1"
@ -4679,27 +4347,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "695a9954f5aa273e44c497c19f806177f787ccf87cd4b3044c96a5057266a861" checksum = "695a9954f5aa273e44c497c19f806177f787ccf87cd4b3044c96a5057266a861"
dependencies = [ dependencies = [
"bytemuck", "bytemuck",
"polars-arrow 0.39.1", "polars-arrow",
"polars-error 0.39.1", "polars-error",
"polars-utils 0.39.1", "polars-utils",
]
[[package]]
name = "polars-sql"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3652c362959f608d1297196b973d1e3acb508a9562b886ac39bf7606b841052b"
dependencies = [
"hex",
"polars-arrow 0.38.3",
"polars-core 0.38.3",
"polars-error 0.38.3",
"polars-lazy 0.38.3",
"polars-plan 0.38.3",
"rand",
"serde",
"serde_json",
"sqlparser 0.39.0",
] ]
[[package]] [[package]]
@ -4709,38 +4359,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7cdf3b41bda70004ed3ec78652eb690aec3db5d99dfac03fbf9995fe76a7e26" checksum = "f7cdf3b41bda70004ed3ec78652eb690aec3db5d99dfac03fbf9995fe76a7e26"
dependencies = [ dependencies = [
"hex", "hex",
"polars-arrow 0.39.1", "polars-arrow",
"polars-core 0.39.1", "polars-core",
"polars-error 0.39.1", "polars-error",
"polars-lazy 0.39.1", "polars-lazy",
"polars-plan 0.39.1", "polars-plan",
"rand", "rand",
"serde", "serde",
"serde_json", "serde_json",
"sqlparser 0.39.0", "sqlparser 0.39.0",
] ]
[[package]]
name = "polars-time"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86eb74ea6ddfe675aa5c3f33c00dadbe2b85f0e8e3887b85db1fd5a3397267fd"
dependencies = [
"atoi",
"chrono",
"chrono-tz 0.8.6",
"now",
"once_cell",
"polars-arrow 0.38.3",
"polars-core 0.38.3",
"polars-error 0.38.3",
"polars-ops 0.38.3",
"polars-utils 0.38.3",
"regex",
"serde",
"smartstring",
]
[[package]] [[package]]
name = "polars-time" name = "polars-time"
version = "0.39.1" version = "0.39.1"
@ -4752,36 +4381,16 @@ dependencies = [
"chrono-tz 0.8.6", "chrono-tz 0.8.6",
"now", "now",
"once_cell", "once_cell",
"polars-arrow 0.39.1", "polars-arrow",
"polars-core 0.39.1", "polars-core",
"polars-error 0.39.1", "polars-error",
"polars-ops 0.39.1", "polars-ops",
"polars-utils 0.39.1", "polars-utils",
"regex", "regex",
"serde", "serde",
"smartstring", "smartstring",
] ]
[[package]]
name = "polars-utils"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "694656a7d2b0cd8f07660dbc8d0fb7a81066ff57a452264907531d805c1e58c4"
dependencies = [
"ahash 0.8.11",
"bytemuck",
"hashbrown 0.14.3",
"indexmap",
"num-traits",
"once_cell",
"polars-error 0.38.3",
"raw-cpuid",
"rayon",
"smartstring",
"sysinfo",
"version_check",
]
[[package]] [[package]]
name = "polars-utils" name = "polars-utils"
version = "0.39.1" version = "0.39.1"
@ -4794,7 +4403,7 @@ dependencies = [
"indexmap", "indexmap",
"num-traits", "num-traits",
"once_cell", "once_cell",
"polars-error 0.39.1", "polars-error",
"raw-cpuid", "raw-cpuid",
"rayon", "rayon",
"smartstring", "smartstring",

12
crates/nu-cmd-dataframe/Cargo.toml
View file

@ -26,11 +26,11 @@ num = { version = "0.4", optional = true }
serde = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] }
# keep sqlparser at 0.39.0 until we can update polars # keep sqlparser at 0.39.0 until we can update polars
sqlparser = { version = "0.45", optional = true } sqlparser = { version = "0.45", optional = true }
polars-io = { version = "0.38", features = ["avro"], optional = true } polars-io = { version = "0.39", features = ["avro"], optional = true }
polars-arrow = { version = "0.38", optional = true } polars-arrow = { version = "0.39", optional = true }
polars-ops = { version = "0.38", optional = true } polars-ops = { version = "0.39", optional = true }
polars-plan = { version = "0.38", features = ["regex"], optional = true } polars-plan = { version = "0.39", features = ["regex"], optional = true }
polars-utils = { version = "0.38", optional = true } polars-utils = { version = "0.39", optional = true }
[dependencies.polars] [dependencies.polars]
features = [ features = [
@ -65,7 +65,7 @@ features = [
] ]
default-features = false default-features = false
optional = true optional = true
version = "0.38" version = "0.39"
[features] [features]
dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "polars-utils", "sqlparser"] dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "polars-utils", "sqlparser"]

7
crates/nu-cmd-dataframe/src/dataframe/eager/open.rs
View file

@ -5,7 +5,7 @@ use polars::prelude::{
CsvEncoding, CsvReader, IpcReader, JsonFormat, JsonReader, LazyCsvReader, LazyFileListReader, CsvEncoding, CsvReader, IpcReader, JsonFormat, JsonReader, LazyCsvReader, LazyFileListReader,
LazyFrame, ParallelStrategy, ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader, LazyFrame, ParallelStrategy, ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader,
}; };
use polars_io::avro::AvroReader; use polars_io::{avro::AvroReader, HiveOptions};
use std::{fs::File, io::BufReader, path::PathBuf}; use std::{fs::File, io::BufReader, path::PathBuf};
#[derive(Clone)] #[derive(Clone)]
@ -151,7 +151,7 @@ fn from_parquet(
low_memory: false, low_memory: false,
cloud_options: None, cloud_options: None,
use_statistics: false, use_statistics: false,
hive_partitioning: false, hive_options: HiveOptions::default(),
}; };
let df: NuLazyFrame = LazyFrame::scan_parquet(file, args) let df: NuLazyFrame = LazyFrame::scan_parquet(file, args)
@ -246,7 +246,8 @@ fn from_ipc(
cache: true, cache: true,
rechunk: false, rechunk: false,
row_index: None, row_index: None,
memmap: true, memory_map: true,
cloud_options: None,
}; };
let df: NuLazyFrame = LazyFrame::scan_ipc(file, args) let df: NuLazyFrame = LazyFrame::scan_ipc(file, args)

11
crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs
View file

@ -1,5 +1,6 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*; use nu_engine::command_prelude::*;
use polars::chunked_array::ops::SortMultipleOptions;
#[derive(Clone)] #[derive(Clone)]
pub struct LazySortBy; pub struct LazySortBy;
@ -126,11 +127,17 @@ impl Command for LazySortBy {
None => expressions.iter().map(|_| false).collect::<Vec<bool>>(), None => expressions.iter().map(|_| false).collect::<Vec<bool>>(),
}; };
let sort_options = SortMultipleOptions {
descending: reverse,
nulls_last,
multithreaded: true,
maintain_order,
};
let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
let lazy = NuLazyFrame::new( let lazy = NuLazyFrame::new(
lazy.from_eager, lazy.from_eager,
lazy.into_polars() lazy.into_polars().sort_by_exprs(&expressions, sort_options),
.sort_by_exprs(&expressions, reverse, nulls_last, maintain_order),
); );
Ok(PipelineData::Value( Ok(PipelineData::Value(

15
crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs
View file

@ -9,7 +9,10 @@ pub use operations::Axis;
use super::{nu_schema::NuSchema, utils::DEFAULT_ROWS, NuLazyFrame}; use super::{nu_schema::NuSchema, utils::DEFAULT_ROWS, NuLazyFrame};
use indexmap::IndexMap; use indexmap::IndexMap;
use nu_protocol::{did_you_mean, PipelineData, Record, ShellError, Span, Value}; use nu_protocol::{did_you_mean, PipelineData, Record, ShellError, Span, Value};
use polars::prelude::{DataFrame, DataType, IntoLazy, LazyFrame, PolarsObject, Series}; use polars::{
chunked_array::ops::SortMultipleOptions,
prelude::{DataFrame, DataType, IntoLazy, LazyFrame, PolarsObject, Series},
};
use polars_plan::prelude::{lit, Expr, Null}; use polars_plan::prelude::{lit, Expr, Null};
use polars_utils::total_ord::{TotalEq, TotalHash}; use polars_utils::total_ord::{TotalEq, TotalHash};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -488,12 +491,18 @@ impl NuDataFrame {
.expect("already checked that dataframe is different than 0"); .expect("already checked that dataframe is different than 0");
// if unable to sort, then unable to compare // if unable to sort, then unable to compare
let lhs = match self.as_ref().sort(vec![*first_col], false, false) { let lhs = match self
.as_ref()
.sort(vec![*first_col], SortMultipleOptions::default())
{
Ok(df) => df, Ok(df) => df,
Err(_) => return None, Err(_) => return None,
}; };
let rhs = match other.as_ref().sort(vec![*first_col], false, false) { let rhs = match other
.as_ref()
.sort(vec![*first_col], SortMultipleOptions::default())
{
Ok(df) => df, Ok(df) => df,
Err(_) => return None, Err(_) => return None,
}; };

8
crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs
View file

@ -313,11 +313,15 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Result<Value, ShellError> {
Expr::SortBy { Expr::SortBy {
expr, expr,
by, by,
descending, sort_options,
} => { } => {
let by: Result<Vec<Value>, ShellError> = let by: Result<Vec<Value>, ShellError> =
by.iter().map(|b| expr_to_value(b, span)).collect(); by.iter().map(|b| expr_to_value(b, span)).collect();
let descending: Vec<Value> = descending.iter().map(|r| Value::bool(*r, span)).collect(); let descending: Vec<Value> = sort_options
.descending
.iter()
.map(|r| Value::bool(*r, span))
.collect();
Ok(Value::record( Ok(Value::record(
record! { record! {