Updating polars and sqlparser versions (#10114)

Polars and SQLParser upgrade.

I have exposed features that have been added to polars as command args
where appropriate.

---------

Co-authored-by: Jack Wright <jack.wright@disqo.com>
Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com>
Co-authored-by: sholderbach <sholderbach@users.noreply.github.com>
This commit is contained in:
Jack Wright 2023-08-29 15:13:34 -07:00 committed by GitHub
parent 3f2c76df28
commit 3fd1a26ec0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 236 additions and 236 deletions

185
Cargo.lock generated
View file

@ -166,9 +166,9 @@ dependencies = [
[[package]]
name = "arrow2"
version = "0.17.2"
version = "0.17.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15ae0428d69ab31d7b2adad22a752d6f11fef2e901d2262d0cad4f5cb08b7093"
checksum = "59c468daea140b747d781a1da9f7db5f0a8e6636d4af20cc539e43d05b0604fa"
dependencies = [
"ahash",
"arrow-format",
@ -370,6 +370,9 @@ name = "bitflags"
version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
dependencies = [
"serde",
]
[[package]]
name = "block-buffer"
@ -679,9 +682,9 @@ dependencies = [
[[package]]
name = "comfy-table"
version = "6.2.0"
version = "7.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e959d788268e3bf9d35ace83e81b124190378e4c91c9067524675e33394b8ba"
checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b"
dependencies = [
"crossterm",
"strum 0.24.1",
@ -888,7 +891,7 @@ dependencies = [
"crossterm_winapi",
"libc",
"mio",
"parking_lot 0.12.1",
"parking_lot",
"serde",
"signal-hook",
"signal-hook-mio",
@ -984,7 +987,7 @@ dependencies = [
"hashbrown 0.14.0",
"lock_api",
"once_cell",
"parking_lot_core 0.9.8",
"parking_lot_core",
]
[[package]]
@ -1597,7 +1600,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
dependencies = [
"ahash",
"rayon",
]
[[package]]
@ -1608,6 +1610,7 @@ checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
dependencies = [
"ahash",
"allocator-api2",
"rayon",
]
[[package]]
@ -1793,7 +1796,6 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown 0.12.3",
"serde",
]
[[package]]
@ -1804,6 +1806,7 @@ checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
dependencies = [
"equivalent",
"hashbrown 0.14.0",
"serde",
]
[[package]]
@ -2310,9 +2313,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "memmap2"
version = "0.5.10"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327"
checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6"
dependencies = [
"libc",
]
@ -2429,9 +2432,9 @@ dependencies = [
[[package]]
name = "multiversion"
version = "0.7.2"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cda45dade5144c2c929bf2ed6c24bebbba784e9198df049ec87d722b9462bd1"
checksum = "b2c7b9d7fe61760ce5ea19532ead98541f6b4c495d87247aff9826445cf6872a"
dependencies = [
"multiversion-macros",
"target-features",
@ -2439,9 +2442,9 @@ dependencies = [
[[package]]
name = "multiversion-macros"
version = "0.7.2"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04bffdccbd4798b61dce08c97ce8c66a68976f95541aaf284a6e90c1d1c306e1"
checksum = "26a83d8500ed06d68877e9de1dde76c1dbb83885dcdbda4ef44ccbc3fbda2ac8"
dependencies = [
"proc-macro2",
"quote",
@ -2534,7 +2537,7 @@ checksum = "416969970ec751a5d702a88c6cd19ac1332abe997fce43f96db0418550426241"
dependencies = [
"file-id",
"notify",
"parking_lot 0.12.1",
"parking_lot",
"walkdir",
]
@ -2806,7 +2809,6 @@ dependencies = [
"serde_urlencoded",
"serde_yaml",
"sha2",
"sqlparser",
"sysinfo",
"tabled",
"terminal_size 0.2.6",
@ -3371,17 +3373,6 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.6",
]
[[package]]
name = "parking_lot"
version = "0.12.1"
@ -3389,21 +3380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api",
"parking_lot_core 0.9.8",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall 0.2.16",
"smallvec",
"winapi",
"parking_lot_core",
]
[[package]]
@ -3633,9 +3610,9 @@ dependencies = [
[[package]]
name = "polars"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c464be75f4ec8b1a8527a55583f67961e3514f112a99f789476a7ccb9a397d10"
checksum = "b1362d4a136c0ebacb40d88a37ba361738b222fd8a2ee9340a3d8642f698c52b"
dependencies = [
"getrandom",
"polars-core",
@ -3649,33 +3626,34 @@ dependencies = [
[[package]]
name = "polars-arrow"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f187a7fd8fbef04d6a810bfd4c181baf584fc4439ec054e0caad4325bf00f95"
checksum = "f967c901fa5da4ca7f64e813d1268488ba97e9b3004cefc579ff851c197a1138"
dependencies = [
"arrow2",
"hashbrown 0.13.2",
"hashbrown 0.14.0",
"multiversion",
"num-traits",
"polars-error",
"serde",
"thiserror",
"version_check",
]
[[package]]
name = "polars-core"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78b7216aa3336fd2a7b5ebfa66748f3770682b28cd682930d1abf59b53b670e0"
checksum = "b24f92fc5b167f668ff85ab9607dfa72e2c09664cacef59297ee8601dee60126"
dependencies = [
"ahash",
"arrow2",
"bitflags 1.3.2",
"bitflags 2.3.3",
"chrono",
"comfy-table",
"either",
"hashbrown 0.13.2",
"indexmap 1.9.3",
"hashbrown 0.14.0",
"indexmap 2.0.0",
"num-traits",
"once_cell",
"polars-arrow",
@ -3690,15 +3668,15 @@ dependencies = [
"serde_json",
"smartstring",
"thiserror",
"wasm-timer",
"version_check",
"xxhash-rust",
]
[[package]]
name = "polars-error"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfb4f020b2b6d12408cca6b081dbd928fef8bbe9cdbba4c88441e61ade33bc2d"
checksum = "40d09c3a7337e53b38c37b57999038440fa39c6801b9ba48afaecd8e16f7ac0a"
dependencies = [
"arrow2",
"regex",
@ -3707,9 +3685,9 @@ dependencies = [
[[package]]
name = "polars-io"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "270650704e08bef37d227a6904b36c4bdad2d013536ea3c57f40007c19ad2ebc"
checksum = "92cab0df9f2a35702fa5aec99edfaabf9ae8e9cdd0acf69e143ad2d132f34f9c"
dependencies = [
"ahash",
"arrow2",
@ -3742,15 +3720,15 @@ dependencies = [
[[package]]
name = "polars-json"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7370203bca4ee29da6da4210200894eea6abf81d7c430e192159dabcba8441a4"
checksum = "9e58094557cf6446808c7348dcb797885db61815857f6ea02924b35505566e94"
dependencies = [
"ahash",
"arrow2",
"fallible-streaming-iterator",
"hashbrown 0.13.2",
"indexmap 1.9.3",
"hashbrown 0.14.0",
"indexmap 2.0.0",
"num-traits",
"polars-arrow",
"polars-error",
@ -3760,12 +3738,12 @@ dependencies = [
[[package]]
name = "polars-lazy"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2a8fe41263496b5212098f19d0cdddc11f75c71957a09d2ce300a8fdb4407e3"
checksum = "2c33762ec2a55e01c9f8776b34db86257c70a0a3b3929bd4eb91a52aacf61456"
dependencies = [
"ahash",
"bitflags 1.3.2",
"bitflags 2.3.3",
"glob",
"once_cell",
"polars-arrow",
@ -3779,33 +3757,36 @@ dependencies = [
"polars-utils",
"rayon",
"smartstring",
"version_check",
]
[[package]]
name = "polars-ops"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d63c31565b88f31457abafb224166fdc0824bf2954414f0c5c52b31c09189942"
checksum = "e825575c96302d2daedfc205a0062180033c92c55bcd6aafc4e109d4d8849ed0"
dependencies = [
"argminmax",
"arrow2",
"either",
"indexmap 2.0.0",
"memchr",
"polars-arrow",
"polars-core",
"polars-utils",
"serde",
"smartstring",
"version_check",
]
[[package]]
name = "polars-pipe"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51076665e7e45f536a2f080763db040e38eab9fdb1cd927b93428720ba23c46a"
checksum = "1f2bc9a12da9ed043fb0cb51dbcb87b365e4845b7ab6399d7a81e838460c6974"
dependencies = [
"enum_dispatch",
"hashbrown 0.13.2",
"hashbrown 0.14.0",
"num-traits",
"polars-arrow",
"polars-core",
@ -3816,13 +3797,14 @@ dependencies = [
"polars-utils",
"rayon",
"smartstring",
"version_check",
]
[[package]]
name = "polars-plan"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a7d0f48bdd7fa9474f718ecb99fc12d97671933707091a249bb0da96b30f291"
checksum = "fb67b014f0295e8e9dbb84404a91d666d477b3bc248a2ed51bc442833b16da35"
dependencies = [
"ahash",
"arrow2",
@ -3837,13 +3819,15 @@ dependencies = [
"regex",
"serde",
"smartstring",
"strum_macros 0.25.1",
"version_check",
]
[[package]]
name = "polars-row"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b18fa58195b88fdd32c98561cbeff4024b106cdd2a8c975e7d06544324d0e4d1"
checksum = "27f54c1956027bf6301948fb4f2837cf6d6b638d8dd1edf3aaeaa19906a986be"
dependencies = [
"arrow2",
"polars-error",
@ -3852,9 +3836,9 @@ dependencies = [
[[package]]
name = "polars-sql"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8485d0c9664e67123b065c8a7c946a0165b0192c05c24cdcde447b1c9e05c7e5"
checksum = "dbfcb15cf8eebd25ea1724109d0153817cd484c6326290585f0736b4e7fcf2f4"
dependencies = [
"polars-arrow",
"polars-core",
@ -3867,9 +3851,9 @@ dependencies = [
[[package]]
name = "polars-time"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c0d11ef8a93b2d204a60da7099510c40fe1ac04d95022ae4daa39df31b345e1"
checksum = "53f42d2632f5971c9575041d33cbcfb1f996900c40bbf58bc6eb0a0c5efbecea"
dependencies = [
"arrow2",
"atoi",
@ -3887,16 +3871,19 @@ dependencies = [
[[package]]
name = "polars-utils"
version = "0.30.0"
version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "145a59f928f8317fcf543407ef1b83e827448462e05e2fc6444162fcec84386a"
checksum = "c326708a370d71dc6e11a8f4bbc10a8479e1c314dc048ba73543b815cd0bf339"
dependencies = [
"ahash",
"hashbrown 0.13.2",
"hashbrown 0.14.0",
"num-traits",
"once_cell",
"polars-error",
"rayon",
"smartstring",
"sysinfo",
"version_check",
]
[[package]]
@ -4631,7 +4618,7 @@ dependencies = [
"futures",
"lazy_static",
"log",
"parking_lot 0.12.1",
"parking_lot",
"serial_test_derive",
]
@ -4824,12 +4811,11 @@ dependencies = [
[[package]]
name = "sqlparser"
version = "0.34.0"
version = "0.36.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37d3706eefb17039056234df6b566b0014f303f867f2656108334a55b8096f59"
checksum = "2eaa1e88e78d2c2460d78b7dc3f0c08dbb606ab4222f9aff36f420d36e307d87"
dependencies = [
"log",
"serde",
]
[[package]]
@ -4873,7 +4859,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"
dependencies = [
"new_debug_unreachable",
"once_cell",
"parking_lot 0.12.1",
"parking_lot",
"phf_shared 0.10.0",
"precomputed-hash",
"serde",
@ -5238,7 +5224,7 @@ dependencies = [
"libc",
"mio",
"num_cpus",
"parking_lot 0.12.1",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
@ -5627,18 +5613,6 @@ dependencies = [
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03"
dependencies = [
"cfg-if",
"js-sys",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.87"
@ -5668,21 +5642,6 @@ version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
[[package]]
name = "wasm-timer"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f"
dependencies = [
"futures",
"js-sys",
"parking_lot 0.11.2",
"pin-utils",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "wax"
version = "0.5.0"

View file

@ -23,8 +23,8 @@ fancy-regex = "0.11"
indexmap = { version = "2.0" }
num = { version = "0.4", optional = true }
serde = { version = "1.0", features = ["derive"] }
sqlparser = { version = "0.34", features = ["serde"], optional = true }
polars-io = { version = "0.30.0", features = ["avro"] }
sqlparser = { version = "0.36.1", optional = true }
polars-io = { version = "0.32", features = ["avro"] }
[dependencies.polars]
features = [
@ -54,7 +54,7 @@ features = [
"to_dummies",
]
optional = true
version = "0.30.0"
version = "0.32"
[features]
dataframe = ["num", "polars", "sqlparser"]

View file

@ -20,6 +20,7 @@ impl Command for Dummies {
fn signature(&self) -> Signature {
Signature::build(self.name())
.switch("drop-first", "Drop first row", Some('d'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
@ -115,10 +116,11 @@ fn command(
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let drop_first: bool = call.has_flag("drop-first");
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
df.as_ref()
.to_dummies(None)
.to_dummies(None, drop_first)
.map_err(|e| {
ShellError::GenericError(
"Error calculating dummies".into(),

View file

@ -128,7 +128,7 @@ fn dataframe_command(
}
fn expression_command(call: &Call, input: Value) -> Result<PipelineData, ShellError> {
let expr = NuExpression::try_from_value(input)?;
let value = expr.to_value(call.head);
let value = expr.to_value(call.head)?;
Ok(PipelineData::Value(value, None))
}

View file

@ -95,10 +95,8 @@ impl Command for ExprOtherwise {
let value = input.into_value(call.head);
let complete: NuExpression = match NuWhen::try_from_value(value)? {
NuWhen::WhenThen(when_then) => when_then
.otherwise(otherwise_predicate.into_polars())
.into(),
NuWhen::WhenThenThen(when_then_then) => when_then_then
NuWhen::Then(then) => then.otherwise(otherwise_predicate.into_polars()).into(),
NuWhen::ChainedThen(chained_when) => chained_when
.otherwise(otherwise_predicate.into_polars())
.into(),
};

View file

@ -110,11 +110,11 @@ impl Command for ExprWhen {
.then(then_predicate.into_polars())
.into(),
v => match NuWhen::try_from_value(v)? {
NuWhen::WhenThen(when_then) => when_then
NuWhen::Then(when_then) => when_then
.when(when_predicate.into_polars())
.then(then_predicate.into_polars())
.into(),
NuWhen::WhenThenThen(when_then_then) => when_then_then
NuWhen::ChainedThen(when_then_then) => when_then_then
.when(when_predicate.into_polars())
.then(then_predicate.into_polars())
.into(),

View file

@ -172,7 +172,6 @@ fn get_col_name(expr: &Expr) -> Option<String> {
},
Expr::Filter { input: expr, .. }
| Expr::Slice { input: expr, .. }
| Expr::Cache { input: expr, .. }
| Expr::Cast { expr, .. }
| Expr::Sort { expr, .. }
| Expr::Take { expr, .. }
@ -192,7 +191,8 @@ fn get_col_name(expr: &Expr) -> Option<String> {
| Expr::Wildcard
| Expr::RenameAlias { .. }
| Expr::Count
| Expr::Nth(_) => None,
| Expr::Nth(_)
| Expr::Selector(_) => None,
}
}

View file

@ -37,6 +37,7 @@ impl Command for LazySortBy {
"nulls are shown last in the dataframe",
Some('n'),
)
.switch("maintain-order", "Maintains order during sort", Some('m'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
@ -110,6 +111,7 @@ impl Command for LazySortBy {
};
let expressions = NuExpression::extract_exprs(value)?;
let nulls_last = call.has_flag("nulls-last");
let maintain_order = call.has_flag("maintain-order");
let reverse: Option<Vec<bool>> = call.get_flag(engine_state, stack, "reverse")?;
let reverse = match reverse {
@ -137,7 +139,7 @@ impl Command for LazySortBy {
let lazy = NuLazyFrame::new(
lazy.from_eager,
lazy.into_polars()
.sort_by_exprs(&expressions, reverse, nulls_last),
.sort_by_exprs(&expressions, reverse, nulls_last, maintain_order),
);
Ok(PipelineData::Value(

View file

@ -143,7 +143,13 @@ fn command(
})?;
let res = if not_exact {
casted.as_datetime_not_exact(Some(format.as_str()), TimeUnit::Nanoseconds, None)
casted.as_datetime_not_exact(
Some(format.as_str()),
TimeUnit::Nanoseconds,
false,
None,
None,
)
} else {
casted.as_datetime(
Some(format.as_str()),
@ -151,6 +157,7 @@ fn command(
false,
false,
None,
None,
)
};

View file

@ -27,6 +27,11 @@ impl Command for ArgSort {
Signature::build(self.name())
.switch("reverse", "reverse order", Some('r'))
.switch("nulls-last", "nulls ordered last", Some('n'))
.switch(
"maintain-order",
"maintain order on sorted items",
Some('m'),
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
@ -98,6 +103,7 @@ fn command(
descending: call.has_flag("reverse"),
nulls_last: call.has_flag("nulls-last"),
multithreaded: true,
maintain_order: call.has_flag("maintain-order"),
};
let mut res = df

View file

@ -98,7 +98,12 @@ fn command(
let value = NuDataFrame::dataframe_into_value(res, call.head);
Ok(PipelineData::Value(value, None))
}
_ => todo!(),
_ => Err(ShellError::UnsupportedInput(
"Expected the dataframe to have a column".to_string(),
"".to_string(),
call.head,
call.head,
)),
}
}

View file

@ -149,6 +149,7 @@ fn command(
closed_window: None,
tu: None,
tz: None,
fn_params: None,
};
let res = match roll_type {
RollType::Max => series.rolling_max(rolling_opts),

View file

@ -421,7 +421,17 @@ fn input_type_list_to_series(
let dt_chunked = ChunkedArray::<Int64Type>::from_iter_options(&list_name, it)
.into_datetime(TimeUnit::Nanoseconds, None);
builder.append_series(&dt_chunked.into_series());
builder
.append_series(&dt_chunked.into_series())
.map_err(|e| {
ShellError::GenericError(
"Error appending to series".into(),
"".to_string(),
None,
Some(e.to_string()),
Vec::new(),
)
})?
}
let res = builder.finish();
Ok(res.into_series())

View file

@ -466,12 +466,12 @@ impl NuDataFrame {
.expect("already checked that dataframe is different than 0");
// if unable to sort, then unable to compare
let lhs = match self.as_ref().sort(vec![*first_col], false) {
let lhs = match self.as_ref().sort(vec![*first_col], false, false) {
Ok(df) => df,
Err(_) => return None,
};
let rhs = match other.as_ref().sort(vec![*first_col], false) {
let rhs = match other.as_ref().sort(vec![*first_col], false, false) {
Ok(df) => df,
Err(_) => return None,
};

View file

@ -31,7 +31,7 @@ impl CustomValue for NuExpression {
}
fn to_base_value(&self, span: Span) -> Result<Value, ShellError> {
Ok(self.to_value(span))
self.to_value(span)
}
fn as_any(&self) -> &dyn std::any::Any {

View file

@ -115,7 +115,7 @@ impl NuExpression {
f(expr, other).into()
}
pub fn to_value(&self, span: Span) -> Value {
pub fn to_value(&self, span: Span) -> Result<Value, ShellError> {
expr_to_value(self.as_ref(), span)
}
@ -164,60 +164,59 @@ impl ExtractedExpr {
}
}
pub fn expr_to_value(expr: &Expr, span: Span) -> Value {
pub fn expr_to_value(expr: &Expr, span: Span) -> Result<Value, ShellError> {
match expr {
Expr::Alias(expr, alias) => Value::record(
Expr::Alias(expr, alias) => Ok(Value::record(
record! {
"expr" => expr_to_value(expr.as_ref(), span),
"expr" => expr_to_value(expr.as_ref(), span)?,
"alias" => Value::string(alias.as_ref(), span),
},
span,
),
Expr::Column(name) => Value::record(
)),
Expr::Column(name) => Ok(Value::record(
record! {
"expr" => Value::string("column", span),
"value" => Value::string(name.to_string(), span),
},
span,
),
)),
Expr::Columns(columns) => {
let value = columns.iter().map(|col| Value::string(col, span)).collect();
Value::record(
Ok(Value::record(
record! {
"expr" => Value::string("columns", span),
"value" => Value::list(value, span),
},
span,
)
))
}
Expr::Literal(literal) => Value::record(
Expr::Literal(literal) => Ok(Value::record(
record! {
"expr" => Value::string("literal", span),
"value" => Value::string(format!("{literal:?}"), span),
},
span,
),
Expr::BinaryExpr { left, op, right } => Value::record(
)),
Expr::BinaryExpr { left, op, right } => Ok(Value::record(
record! {
"left" => expr_to_value(left, span),
"left" => expr_to_value(left, span)?,
"op" => Value::string(format!("{op:?}"), span),
"right" => expr_to_value(right, span),
"right" => expr_to_value(right, span)?,
},
span,
),
)),
Expr::Ternary {
predicate,
truthy,
falsy,
} => Value::record(
} => Ok(Value::record(
record! {
"predicate" => expr_to_value(predicate.as_ref(), span),
"truthy" => expr_to_value(truthy.as_ref(), span),
"falsy" => expr_to_value(falsy.as_ref(), span),
"predicate" => expr_to_value(predicate.as_ref(), span)?,
"truthy" => expr_to_value(truthy.as_ref(), span)?,
"falsy" => expr_to_value(falsy.as_ref(), span)?,
},
span,
),
)),
Expr::Agg(agg_expr) => {
let value = match agg_expr {
AggExpr::Min { input: expr, .. }
@ -237,37 +236,44 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value {
expr,
quantile,
interpol,
} => Value::record(
} => Ok(Value::record(
record! {
"expr" => expr_to_value(expr.as_ref(), span),
"quantile" => expr_to_value(quantile.as_ref(), span),
"expr" => expr_to_value(expr.as_ref(), span)?,
"quantile" => expr_to_value(quantile.as_ref(), span)?,
"interpol" => Value::string(format!("{interpol:?}"), span),
},
span,
),
)),
};
Value::record(
Ok(Value::record(
record! {
"expr" => Value::string("agg", span),
"value" => value,
"value" => value?,
},
span,
)
))
}
Expr::Count => Value::record(record! { "expr" => Value::string("count", span) }, span),
Expr::Wildcard => {
Value::record(record! { "expr" => Value::string("wildcard", span) }, span)
}
Expr::Explode(expr) => Value::record(
record! { "expr" => expr_to_value(expr.as_ref(), span) },
Expr::Count => Ok(Value::record(
record! { "expr" => Value::string("count", span) },
span,
),
Expr::KeepName(expr) => Value::record(
record! { "expr" => expr_to_value(expr.as_ref(), span) },
)),
Expr::Wildcard => Ok(Value::record(
record! { "expr" => Value::string("wildcard", span) },
span,
),
Expr::Nth(i) => Value::record(record! { "expr" => Value::int(*i, span) }, span),
)),
Expr::Explode(expr) => Ok(Value::record(
record! { "expr" => expr_to_value(expr.as_ref(), span)? },
span,
)),
Expr::KeepName(expr) => Ok(Value::record(
record! { "expr" => expr_to_value(expr.as_ref(), span)? },
span,
)),
Expr::Nth(i) => Ok(Value::record(
record! { "expr" => Value::int(*i, span) },
span,
)),
Expr::DtypeColumn(dtypes) => {
let vals = dtypes
.iter()
@ -277,73 +283,74 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value {
})
.collect();
Value::List { vals, span }
Ok(Value::List { vals, span })
}
Expr::Sort { expr, options } => Value::record(
Expr::Sort { expr, options } => Ok(Value::record(
record! {
"expr" => expr_to_value(expr.as_ref(), span),
"expr" => expr_to_value(expr.as_ref(), span)?,
"options" => Value::string(format!("{options:?}"), span),
},
span,
),
)),
Expr::Cast {
expr,
data_type,
strict,
} => Value::record(
} => Ok(Value::record(
record! {
"expr" => expr_to_value(expr.as_ref(), span),
"expr" => expr_to_value(expr.as_ref(), span)?,
"dtype" => Value::string(format!("{data_type:?}"), span),
"strict" => Value::bool(*strict, span),
},
span,
),
Expr::Take { expr, idx } => Value::record(
)),
Expr::Take { expr, idx } => Ok(Value::record(
record! {
"expr" => expr_to_value(expr.as_ref(), span),
"idx" => expr_to_value(idx.as_ref(), span),
"expr" => expr_to_value(expr.as_ref(), span)?,
"idx" => expr_to_value(idx.as_ref(), span)?,
},
span,
),
)),
Expr::SortBy {
expr,
by,
descending,
} => {
let by: Vec<Value> = by.iter().map(|b| expr_to_value(b, span)).collect();
let by: Result<Vec<Value>, ShellError> =
by.iter().map(|b| expr_to_value(b, span)).collect();
let descending: Vec<Value> = descending
.iter()
.map(|r| Value::Bool { val: *r, span })
.collect();
Value::record(
Ok(Value::record(
record! {
"expr" => expr_to_value(expr.as_ref(), span),
"by" => Value::list(by, span),
"expr" => expr_to_value(expr.as_ref(), span)?,
"by" => Value::list(by?, span),
"descending" => Value::list(descending, span),
},
span,
)
))
}
Expr::Filter { input, by } => Value::record(
Expr::Filter { input, by } => Ok(Value::record(
record! {
"input" => expr_to_value(input.as_ref(), span),
"by" => expr_to_value(by.as_ref(), span),
"input" => expr_to_value(input.as_ref(), span)?,
"by" => expr_to_value(by.as_ref(), span)?,
},
span,
),
)),
Expr::Slice {
input,
offset,
length,
} => Value::record(
} => Ok(Value::record(
record! {
"input" => expr_to_value(input.as_ref(), span),
"offset" => expr_to_value(offset.as_ref(), span),
"length" => expr_to_value(length.as_ref(), span),
"input" => expr_to_value(input.as_ref(), span)?,
"offset" => expr_to_value(offset.as_ref(), span)?,
"length" => expr_to_value(length.as_ref(), span)?,
},
span,
),
)),
Expr::Exclude(expr, excluded) => {
let excluded = excluded
.iter()
@ -353,67 +360,62 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value {
})
.collect();
Value::record(
Ok(Value::record(
record! {
"expr" => expr_to_value(expr.as_ref(), span),
"expr" => expr_to_value(expr.as_ref(), span)?,
"excluded" => Value::list(excluded, span),
},
span,
)
))
}
Expr::RenameAlias { expr, function } => Value::record(
Expr::RenameAlias { expr, function } => Ok(Value::record(
record! {
"expr" => expr_to_value(expr.as_ref(), span),
"expr" => expr_to_value(expr.as_ref(), span)?,
"function" => Value::string(format!("{function:?}"), span),
},
span,
),
)),
Expr::AnonymousFunction {
input,
function,
output_type,
options,
} => {
let input: Vec<Value> = input.iter().map(|e| expr_to_value(e, span)).collect();
Value::record(
let input: Result<Vec<Value>, ShellError> =
input.iter().map(|e| expr_to_value(e, span)).collect();
Ok(Value::record(
record! {
"input" => Value::list(input, span),
"input" => Value::list(input?, span),
"function" => Value::string(format!("{function:?}"), span),
"output_type" => Value::string(format!("{output_type:?}"), span),
"options" => Value::string(format!("{options:?}"), span),
},
span,
)
))
}
Expr::Function {
input,
function,
options,
} => {
let input: Vec<Value> = input.iter().map(|e| expr_to_value(e, span)).collect();
Value::record(
let input: Result<Vec<Value>, ShellError> =
input.iter().map(|e| expr_to_value(e, span)).collect();
Ok(Value::record(
record! {
"input" => Value::list(input, span),
"input" => Value::list(input?, span),
"function" => Value::string(format!("{function:?}"), span),
"options" => Value::string(format!("{options:?}"), span),
},
span,
)
))
}
Expr::Cache { input, id } => Value::record(
record! {
"input" => expr_to_value(input.as_ref(), span),
"id" => Value::string(format!("{id:?}"), span),
},
span,
),
Expr::Window {
function,
partition_by,
order_by,
options,
} => {
let partition_by: Vec<Value> = partition_by
let partition_by: Result<Vec<Value>, ShellError> = partition_by
.iter()
.map(|e| expr_to_value(e, span))
.collect();
@ -421,17 +423,26 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value {
let order_by = order_by
.as_ref()
.map(|e| expr_to_value(e.as_ref(), span))
.transpose()?
.unwrap_or_else(|| Value::nothing(span));
Value::record(
Ok(Value::record(
record! {
"function" => expr_to_value(function, span),
"partition_by" => Value::list(partition_by, span),
"function" => expr_to_value(function, span)?,
"partition_by" => Value::list(partition_by?, span),
"order_by" => order_by,
"options" => Value::string(format!("{options:?}"), span),
},
span,
)
}
))
}
// the parameter polars_plan::dsl::selector::Selector is not publicly exposed.
// I am not sure what we can meaningfully do with this at this time.
Expr::Selector(_) => Err(ShellError::UnsupportedInput(
"Expressions of type Selector to Nu Values is not yet supported".to_string(),
format!("Expression is {expr:?}"),
span,
Span::unknown(),
)),
}
}

View file

@ -26,8 +26,8 @@ impl CustomValue for NuWhen {
fn to_base_value(&self, span: Span) -> Result<Value, ShellError> {
let val = match self {
NuWhen::WhenThen(_) => "whenthen".into(),
NuWhen::WhenThenThen(_) => "whenthenthen".into(),
NuWhen::Then(_) => "whenthen".into(),
NuWhen::ChainedThen(_) => "whenthenthen".into(),
};
let value = Value::String { val, span };

View file

@ -2,13 +2,13 @@ mod custom_value;
use core::fmt;
use nu_protocol::{ShellError, Span, Value};
use polars::prelude::{col, when, WhenThen, WhenThenThen};
use polars::prelude::{col, when, ChainedThen, Then};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Clone)]
pub enum NuWhen {
WhenThen(Box<WhenThen>),
WhenThenThen(WhenThenThen),
Then(Box<Then>),
ChainedThen(ChainedThen),
}
// Mocked serialization of the LazyFrame object
@ -27,7 +27,7 @@ impl<'de> Deserialize<'de> for NuWhen {
where
D: Deserializer<'de>,
{
Ok(NuWhen::WhenThen(Box::new(when(col("a")).then(col("b")))))
Ok(NuWhen::Then(Box::new(when(col("a")).then(col("b")))))
}
}
@ -37,15 +37,15 @@ impl fmt::Debug for NuWhen {
}
}
impl From<WhenThen> for NuWhen {
fn from(when_then: WhenThen) -> Self {
NuWhen::WhenThen(Box::new(when_then))
impl From<Then> for NuWhen {
fn from(then: Then) -> Self {
NuWhen::Then(Box::new(then))
}
}
impl From<WhenThenThen> for NuWhen {
fn from(when_then_then: WhenThenThen) -> Self {
NuWhen::WhenThenThen(when_then_then)
impl From<ChainedThen> for NuWhen {
fn from(chained_when: ChainedThen) -> Self {
NuWhen::ChainedThen(chained_when)
}
}

View file

@ -80,7 +80,6 @@ serde_json = "1.0"
serde_urlencoded = "0.7"
serde_yaml = "0.9"
sha2 = "0.10"
sqlparser = { version = "0.34", features = ["serde"], optional = true }
sysinfo = "0.29"
tabled = { version = "0.14.0", features = ["color"], default-features = false }
terminal_size = "0.2"