Polars 0.38 upgrade (#12506)

# Description
Polars 0.38 upgrade for both the dataframe crate and the polars plugin.

---------

Co-authored-by: Jack Wright <jack.wright@disqo.com>
This commit is contained in:
Jack Wright 2024-04-13 11:00:04 -07:00 committed by GitHub
parent 211d9c685c
commit b9dd47ebb7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 150 additions and 67 deletions

146
Cargo.lock generated
View file

@ -362,7 +362,7 @@ dependencies = [
"bitflags 2.5.0",
"cexpr",
"clang-sys",
"itertools 0.11.0",
"itertools 0.12.1",
"lazy_static",
"lazycell",
"proc-macro2",
@ -691,7 +691,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e"
dependencies = [
"chrono",
"chrono-tz-build",
"chrono-tz-build 0.2.1",
"phf 0.11.2",
]
[[package]]
name = "chrono-tz"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb"
dependencies = [
"chrono",
"chrono-tz-build 0.3.0",
"phf 0.11.2",
]
@ -706,6 +717,17 @@ dependencies = [
"phf_codegen 0.11.2",
]
[[package]]
name = "chrono-tz-build"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1"
dependencies = [
"parse-zoneinfo",
"phf 0.11.2",
"phf_codegen 0.11.2",
]
[[package]]
name = "chumsky"
version = "0.9.3"
@ -2075,6 +2097,17 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "jsonpath_lib_polars_vendor"
version = "0.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4bd9354947622f7471ff713eacaabdb683ccb13bba4edccaab9860abf480b7d"
dependencies = [
"log",
"serde",
"serde_json",
]
[[package]]
name = "kqueue"
version = "1.0.8"
@ -2773,7 +2806,7 @@ name = "nu-cmd-dataframe"
version = "0.92.3"
dependencies = [
"chrono",
"chrono-tz",
"chrono-tz 0.8.6",
"fancy-regex",
"indexmap",
"nu-cmd-lang",
@ -2788,7 +2821,7 @@ dependencies = [
"polars-plan",
"polars-utils",
"serde",
"sqlparser 0.39.0",
"sqlparser 0.45.0",
]
[[package]]
@ -2853,7 +2886,7 @@ dependencies = [
"chardetng",
"chrono",
"chrono-humanize",
"chrono-tz",
"chrono-tz 0.8.6",
"crossterm",
"csv",
"dialoguer",
@ -3239,7 +3272,7 @@ name = "nu_plugin_polars"
version = "0.92.3"
dependencies = [
"chrono",
"chrono-tz",
"chrono-tz 0.9.0",
"fancy-regex",
"indexmap",
"nu-cmd-lang",
@ -3258,7 +3291,7 @@ dependencies = [
"polars-plan",
"polars-utils",
"serde",
"sqlparser 0.43.1",
"sqlparser 0.45.0",
"tempfile",
"typetag",
"uuid",
@ -3806,25 +3839,29 @@ dependencies = [
[[package]]
name = "polars"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e43795c49010cb851d45227caa17769e83760e21d260ba6285c563b754e1652f"
checksum = "f01006048a264047d6cba081fed8e11adbd69c15956f9e53185a9ac4a541853c"
dependencies = [
"getrandom",
"polars-arrow",
"polars-core",
"polars-error",
"polars-io",
"polars-lazy",
"polars-ops",
"polars-parquet",
"polars-sql",
"polars-time",
"polars-utils",
"version_check",
]
[[package]]
name = "polars-arrow"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faacd21a2548fa6d50c72d6b8d4649a8e029a0f3c6c5545b7f436f0610e49b0f"
checksum = "25197f40d71f82b2f79bb394f03e555d3cc1ce4db1dd052c28318721c71e96ad"
dependencies = [
"ahash 0.8.11",
"atoi",
@ -3832,7 +3869,7 @@ dependencies = [
"avro-schema",
"bytemuck",
"chrono",
"chrono-tz",
"chrono-tz 0.8.6",
"dyn-clone",
"either",
"ethnum",
@ -3870,29 +3907,31 @@ dependencies = [
[[package]]
name = "polars-compute"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32d9dc87f8003ae0edeef5ad9ac92b2a345480bbe17adad64496113ae84706dd"
checksum = "c354515f73cdbbad03c2bf723fcd68e6825943b3ec503055abc8a8cb08ce46bb"
dependencies = [
"bytemuck",
"either",
"num-traits",
"polars-arrow",
"polars-error",
"polars-utils",
"strength_reduce",
"version_check",
]
[[package]]
name = "polars-core"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "befd4d280a82219a01035c4f901319ceba65998c594d0c64f9a439cdee1d7777"
checksum = "6f20d3c227186f74aa3c228c64ef72f5a15617322fed30b4323eaf53b25f8e7b"
dependencies = [
"ahash 0.8.11",
"bitflags 2.5.0",
"bytemuck",
"chrono",
"chrono-tz",
"chrono-tz 0.8.6",
"comfy-table",
"either",
"hashbrown 0.14.3",
@ -3918,9 +3957,9 @@ dependencies = [
[[package]]
name = "polars-error"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f2435b02d1ba36d8c1f6a722cad04e4c0b2705a3112c5706e6960d405d7798"
checksum = "d66dd0ce51f8bd620eb8bd376502fe68a2b1a446d5433ecd2e75270b0755ce76"
dependencies = [
"avro-schema",
"polars-arrow-format",
@ -3931,9 +3970,9 @@ dependencies = [
[[package]]
name = "polars-io"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b51fba2cf014cb39c2b38353d601540fb9db643be65abb9ca8ff44b9c4c4a88e"
checksum = "b40bef2edcdc58394792c4d779465144283a09ff1836324e7b72df7978a6e992"
dependencies = [
"ahash 0.8.11",
"async-trait",
@ -3972,9 +4011,9 @@ dependencies = [
[[package]]
name = "polars-json"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "973d1f40ba964e70cf0038779056a7850f649538f72d8828c21bc1a7bce312ed"
checksum = "ef86aca08f10ddc939fe95aabb44e1d2582dcb08b55d4dadb93353ce42adc248"
dependencies = [
"ahash 0.8.11",
"chrono",
@ -3993,9 +4032,9 @@ dependencies = [
[[package]]
name = "polars-lazy"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d83343e413346f048f3a5ad07c0ea4b5d0bada701a482878213142970b0ddff8"
checksum = "c27df26a19d3092298d31d47614ad84dc330c106e38aa8cd53727cd91c07cf56"
dependencies = [
"ahash 0.8.11",
"bitflags 2.5.0",
@ -4017,32 +4056,35 @@ dependencies = [
[[package]]
name = "polars-ops"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6395f5fd5e1adf016fd6403c0a493181c1a349a7a145b2687cdf50a0d630310a"
checksum = "7f8a51c3bdc9e7c34196ff6f5c3cb17da134e5aafb1756aaf24b76c7118e63dc"
dependencies = [
"ahash 0.8.11",
"argminmax",
"base64 0.21.7",
"bytemuck",
"chrono",
"chrono-tz",
"chrono-tz 0.8.6",
"either",
"hashbrown 0.14.3",
"hex",
"indexmap",
"jsonpath_lib_polars_vendor",
"memchr",
"num-traits",
"polars-arrow",
"polars-compute",
"polars-core",
"polars-error",
"polars-json",
"polars-utils",
"rand",
"rand_distr",
"rayon",
"regex",
"serde",
"serde_json",
"smartstring",
"unicode-reverse",
"version_check",
@ -4050,9 +4092,9 @@ dependencies = [
[[package]]
name = "polars-parquet"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b664cac41636cc9f146fba584a8e7c2790d7335a278964529fa3e9b4eae96daf"
checksum = "b8824ee00fbbe83d69553f2711014c50361238d210ed81a7a297695b7db97d42"
dependencies = [
"ahash 0.8.11",
"async-stream",
@ -4076,9 +4118,9 @@ dependencies = [
[[package]]
name = "polars-pipe"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "390a831b864bc57a4cb260b0595030dfb6a4260a3723cf8ca17968ee2078b8ff"
checksum = "0c5e2c1f14e81d60cfa9afe4e611a9bad9631a2cb7cd19b7c0094d0dc32f0231"
dependencies = [
"crossbeam-channel",
"crossbeam-queue",
@ -4095,18 +4137,19 @@ dependencies = [
"polars-utils",
"rayon",
"smartstring",
"uuid",
"version_check",
]
[[package]]
name = "polars-plan"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fb7d7527be2aa33baace9000f6772eb9df7cd57ec010a4b273435d2dc1349e8"
checksum = "ff48362bd1b078bbbec7e7ba9ec01fea58fee2887db22a8e3deaf78f322fa3c4"
dependencies = [
"ahash 0.8.11",
"bytemuck",
"chrono-tz",
"chrono-tz 0.8.6",
"once_cell",
"percent-encoding",
"polars-arrow",
@ -4127,10 +4170,11 @@ dependencies = [
[[package]]
name = "polars-row"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4984d97aad3d0db92afe76ebcab10b5e37a1216618b5703ae0d2917ccd6168c"
checksum = "63029da56ff6a720b190490bbc7b6263f9b72d1134311b1f381fc8d306d37770"
dependencies = [
"bytemuck",
"polars-arrow",
"polars-error",
"polars-utils",
@ -4138,9 +4182,9 @@ dependencies = [
[[package]]
name = "polars-sql"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77f62a8b8f93146ec1eb2ef340d77eeb174e8010035e449bfdd424d2b1fd944a"
checksum = "3652c362959f608d1297196b973d1e3acb508a9562b886ac39bf7606b841052b"
dependencies = [
"hex",
"polars-arrow",
@ -4156,13 +4200,13 @@ dependencies = [
[[package]]
name = "polars-time"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d75348a51d0c97f3b83df860ecb35a6ac6c5dafc6278cac4e1ac101d96dc753"
checksum = "86eb74ea6ddfe675aa5c3f33c00dadbe2b85f0e8e3887b85db1fd5a3397267fd"
dependencies = [
"atoi",
"chrono",
"chrono-tz",
"chrono-tz 0.8.6",
"now",
"once_cell",
"polars-arrow",
@ -4177,9 +4221,9 @@ dependencies = [
[[package]]
name = "polars-utils"
version = "0.37.0"
version = "0.38.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38f9c955bb1e9b55d835aeb7fe4e4e8826e01abe5f0ada979ceb7d2b9af7b569"
checksum = "694656a7d2b0cd8f07660dbc8d0fb7a81066ff57a452264907531d805c1e58c4"
dependencies = [
"ahash 0.8.11",
"bytemuck",
@ -4188,6 +4232,7 @@ dependencies = [
"num-traits",
"once_cell",
"polars-error",
"raw-cpuid",
"rayon",
"smartstring",
"sysinfo",
@ -4510,6 +4555,15 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "raw-cpuid"
version = "11.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1"
dependencies = [
"bitflags 2.5.0",
]
[[package]]
name = "rayon"
version = "1.10.0"
@ -5264,9 +5318,9 @@ dependencies = [
[[package]]
name = "sqlparser"
version = "0.43.1"
version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f95c4bae5aba7cd30bd506f7140026ade63cff5afd778af8854026f9606bf5d4"
checksum = "f7bbffee862a796d67959a89859d6b1046bb5016d63e23835ad0da182777bbe0"
dependencies = [
"log",
]

View file

@ -25,12 +25,12 @@ indexmap = { workspace = true }
num = { version = "0.4", optional = true }
serde = { workspace = true, features = ["derive"] }
# keep sqlparser at 0.39.0 until we can update polars
sqlparser = { version = "0.39.0", optional = true }
polars-io = { version = "0.37", features = ["avro"], optional = true }
polars-arrow = { version = "0.37", optional = true }
polars-ops = { version = "0.37", optional = true }
polars-plan = { version = "0.37", features = ["regex"], optional = true }
polars-utils = { version = "0.37", optional = true }
sqlparser = { version = "0.45", optional = true }
polars-io = { version = "0.38", features = ["avro"], optional = true }
polars-arrow = { version = "0.38", optional = true }
polars-ops = { version = "0.38", optional = true }
polars-plan = { version = "0.38", features = ["regex"], optional = true }
polars-utils = { version = "0.38", optional = true }
[dependencies.polars]
features = [
@ -65,7 +65,7 @@ features = [
]
default-features = false
optional = true
version = "0.37"
version = "0.38"
[features]
dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "polars-utils", "sqlparser"]

View file

@ -11,9 +11,14 @@ use indexmap::IndexMap;
use nu_protocol::{did_you_mean, PipelineData, Record, ShellError, Span, Value};
use polars::prelude::{DataFrame, DataType, IntoLazy, LazyFrame, PolarsObject, Series};
use polars_plan::prelude::{lit, Expr, Null};
use polars_utils::total_ord::TotalEq;
use polars_utils::total_ord::{TotalEq, TotalHash};
use serde::{Deserialize, Serialize};
use std::{cmp::Ordering, collections::HashSet, fmt::Display, hash::Hasher};
use std::{
cmp::Ordering,
collections::HashSet,
fmt::Display,
hash::{Hash, Hasher},
};
// DataFrameValue is an encapsulation of Nushell Value that can be used
// to define the PolarsObject Trait. The polars object trait allows to
@ -31,6 +36,15 @@ impl DataFrameValue {
}
}
impl TotalHash for DataFrameValue {
fn tot_hash<H>(&self, state: &mut H)
where
H: Hasher,
{
(*self).hash(state)
}
}
impl Display for DataFrameValue {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0.get_type())
@ -50,7 +64,7 @@ impl PartialEq for DataFrameValue {
}
impl Eq for DataFrameValue {}
impl std::hash::Hash for DataFrameValue {
impl Hash for DataFrameValue {
fn hash<H: Hasher>(&self, state: &mut H) {
match &self.0 {
Value::Nothing { .. } => 0.hash(state),

View file

@ -23,17 +23,17 @@ nu-path = { path = "../nu-path", version = "0.92.3" }
# Potential dependencies for extras
chrono = { workspace = true, features = ["std", "unstable-locales"], default-features = false }
chrono-tz = "0.8"
chrono-tz = "0.9"
fancy-regex = { workspace = true }
indexmap = { version = "2.2" }
num = {version = "0.4"}
serde = { version = "1.0", features = ["derive"] }
sqlparser = { version = "0.43"}
polars-io = { version = "0.37", features = ["avro"]}
polars-arrow = { version = "0.37"}
polars-ops = { version = "0.37"}
polars-plan = { version = "0.37", features = ["regex"]}
polars-utils = { version = "0.37"}
sqlparser = { version = "0.45"}
polars-io = { version = "0.38", features = ["avro"]}
polars-arrow = { version = "0.38"}
polars-ops = { version = "0.38"}
polars-plan = { version = "0.38", features = ["regex"]}
polars-utils = { version = "0.38"}
typetag = "0.2"
uuid = { version = "1.7", features = ["v4", "serde"] }
@ -69,7 +69,7 @@ features = [
"to_dummies",
]
optional = false
version = "0.37"
version = "0.38"
[dev-dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.92.3" }

View file

@ -10,8 +10,14 @@ use indexmap::map::IndexMap;
use nu_protocol::{did_you_mean, PipelineData, Record, ShellError, Span, Value};
use polars::prelude::{DataFrame, DataType, IntoLazy, PolarsObject, Series};
use polars_plan::prelude::{lit, Expr, Null};
use polars_utils::total_ord::TotalEq;
use std::{cmp::Ordering, collections::HashSet, fmt::Display, hash::Hasher, sync::Arc};
use polars_utils::total_ord::{TotalEq, TotalHash};
use std::{
cmp::Ordering,
collections::HashSet,
fmt::Display,
hash::{Hash, Hasher},
sync::Arc,
};
use uuid::Uuid;
use crate::{Cacheable, PolarsPlugin};
@ -39,6 +45,15 @@ impl DataFrameValue {
}
}
impl TotalHash for DataFrameValue {
fn tot_hash<H>(&self, state: &mut H)
where
H: Hasher,
{
(*self).hash(state)
}
}
impl Display for DataFrameValue {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0.get_type())
@ -58,7 +73,7 @@ impl PartialEq for DataFrameValue {
}
impl Eq for DataFrameValue {}
impl std::hash::Hash for DataFrameValue {
impl Hash for DataFrameValue {
fn hash<H: Hasher>(&self, state: &mut H) {
match &self.0 {
Value::Nothing { .. } => 0.hash(state),