From 5e8754bd8526ee895f1dcfd2fa96132db39ca3f8 Mon Sep 17 00:00:00 2001 From: Hofer-Julian <30049909+Hofer-Julian@users.noreply.github.com> Date: Fri, 12 May 2023 14:44:35 +0200 Subject: [PATCH] Start to move to polars 0.29 (#9145) This does part of the work of porting to polars 0.29. However, I am not familiar enough with this part of the codebase to finish it. Things to be done: - We match two times over `polars::Expr` but `Expr::Cache` isn't handled. I don't know what should be done here - `ArgExpr:::List` was renamed to `ArgExpr::Implode`. Does that mean that `dfr list` should be renamed to `dfr implode`? --------- Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com> --- Cargo.lock | 137 +++++++++++------- crates/nu-command/Cargo.toml | 4 +- .../src/dataframe/eager/sql_context.rs | 2 +- .../expressions/expressions_macro.rs | 6 +- .../src/dataframe/lazy/aggregate.rs | 3 +- .../src/dataframe/values/nu_expression/mod.rs | 17 ++- 6 files changed, 106 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dcae47dec3..ea6a56a72a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,6 +111,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +[[package]] +name = "argminmax" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202108b46429b765ef483f8a24d5c46f48c14acfdacc086dd4ab6dddf6bcdbd2" +dependencies = [ + "num-traits", +] + [[package]] name = "array-init-cursor" version = "0.2.0" @@ -165,6 +174,8 @@ dependencies = [ "multiversion", "num-traits", "parquet2", + "regex", + "regex-syntax", "rustc_version", "simdutf8", "streaming-iterator", @@ -1045,15 +1056,6 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "dirs" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" -dependencies = [ - "dirs-sys", -] - [[package]] name = "dirs-next" version = "2.0.0" @@ -1064,17 +1066,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" -dependencies = [ - "libc", - "redox_users", - "winapi 0.3.9", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -1629,12 +1620,12 @@ checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" [[package]] name = "halfbrown" -version = "0.1.18" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e2a3c70a9c00cc1ee87b54e89f9505f73bb17d63f1b25c9a462ba8ef885444f" +checksum = "6d9ab7d9233262d3b74ae4c4a7a090fc4379b07c6eb9b02ecab7cbb12ad67a58" dependencies = [ - "fxhash", "hashbrown 0.13.2", + "rustc-hash", "serde", ] @@ -1740,6 +1731,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "html5ever" version = "0.26.0" @@ -2873,7 +2873,7 @@ dependencies = [ "serde_urlencoded", "serde_yaml", "sha2", - "sqlparser", + "sqlparser 0.32.0", "sysinfo", "tabled", "terminal_size 0.2.6", @@ -3757,24 +3757,25 @@ dependencies = [ [[package]] name = "polars" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8f04742abbadbd348d73a803617e6df61464485a410361e9c6634e55a4bc01c" +checksum = "aa52642dac73150dcb6ccea276344b2f3c671912b78187ef917e236010ff72ea" dependencies = [ "getrandom 0.2.9", "polars-core", "polars-io", "polars-lazy", "polars-ops", + "polars-sql", "polars-time", "version_check", ] [[package]] name = "polars-arrow" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eacd179fe0cc7cf18e12a80cfda3e6d15ce907655203cc69277e883a925f9bd8" +checksum = "3d2402d68b447ece4afca6bcac435ff0aa82d70d41c9dd3f0ed249396b41ed5b" dependencies = [ "arrow2", "hashbrown 0.13.2", @@ -3787,9 +3788,9 @@ dependencies = [ [[package]] name = "polars-core" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04145d95791f31a0df3b2d3d600cbb5d0d43ccc105d74e871ffe7351a5e70cbe" +checksum = "caa1f5af0505c67333487c1287f29644c172afbab7374e48a77843646064411c" dependencies = [ "ahash 0.8.3", "arrow2", @@ -3819,9 +3820,9 @@ dependencies = [ [[package]] name = "polars-error" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc2fb4ff74f7224b4096b3551b309c50bfd49bb0b66fb3948e70feffb1a8e51" +checksum = "73c760a2564a930a0b139d08045b7f2a3c0c86ce2668710b44561091d8d1e019" dependencies = [ "arrow2", "regex", @@ -3830,18 +3831,18 @@ dependencies = [ [[package]] name = "polars-io" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d186157d427b39085f812e74eefb04608b7bd033ef3a7f848b93a5939c396d" +checksum = "940e27258666c1d5d928d8b086a241a7a77bf04f370883aaf6c57ff2e83e13b8" dependencies = [ "ahash 0.8.3", "arrow2", "async-trait", "bytes", "chrono", - "dirs", "fast-float", "futures", + "home", "lexical", "lexical-core", "memchr", @@ -3864,13 +3865,14 @@ dependencies = [ [[package]] name = "polars-lazy" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385b1beb0ae4f1c8fbe6aef9769b8e0a22a8b56ba494e382f17de82ce653702f" +checksum = "e01ffc3402e9ad40f72d7b8baff7f05fcd12ecd8d19596084bfaa76d9c4e540e" dependencies = [ "ahash 0.8.3", "bitflags", "glob", + "once_cell", "polars-arrow", "polars-core", "polars-io", @@ -3885,10 +3887,11 @@ dependencies = [ [[package]] name = "polars-ops" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8beae89044ad6f5a2237103852d2650c7a52e7b339e356c1d707b9427b48ba7" +checksum = "192e1871522f2c5b1161f7a51432f27389da751132dc3a472ac77ca213186ae1" dependencies = [ + "argminmax", "arrow2", "either", "memchr", @@ -3901,11 +3904,10 @@ dependencies = [ [[package]] name = "polars-pipe" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59c7fbda6ff77db1ae1a0d8f210b9729b2c0cb7c584345504a8282e3bd38e97d" +checksum = "d523fe77338a05266884210633f7bd372214cf7277cbf1d7f8133ab2a11978d9" dependencies = [ - "crossbeam-channel", "enum_dispatch", "hashbrown 0.13.2", "num-traits", @@ -3922,9 +3924,9 @@ dependencies = [ [[package]] name = "polars-plan" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b788cb03c651c9b92c2293b0adc07cb89ae64d7f187310cb7d62d688ccd9d8" +checksum = "7354206859d61710cf85529609ac39885907618bd8160f24ced92112aa36ada5" dependencies = [ "ahash 0.8.3", "arrow2", @@ -3936,15 +3938,16 @@ dependencies = [ "polars-time", "polars-utils", "rayon", + "regex", "serde", "smartstring", ] [[package]] name = "polars-row" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bfcd216b7953321a6946c38d352fdfa67260a4d2125cbb4a1c3b46f03d0bebb" +checksum = "bbe1bda56117570fc1efd79b23fe40ab2375939387c6aaa048d181f0c14fb6b8" dependencies = [ "arrow2", "polars-error", @@ -3952,10 +3955,25 @@ dependencies = [ ] [[package]] -name = "polars-time" -version = "0.28.0" +name = "polars-sql" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5384945cc104ba99683455760dcfe1048e4025bfaf78043e14450cc8c988a745" +checksum = "9520f4e3b7c856f2ea1fc3193b8b7232b82c697b7e1e8098b572bbdb7b007292" +dependencies = [ + "polars-arrow", + "polars-core", + "polars-lazy", + "polars-plan", + "serde", + "serde_json", + "sqlparser 0.30.0", +] + +[[package]] +name = "polars-time" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7406b06f575e8c6e9cc28b97112fd37424262f5eacf3cc95c110381c82ad2dfe" dependencies = [ "arrow2", "atoi", @@ -3973,9 +3991,9 @@ dependencies = [ [[package]] name = "polars-utils" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c1823141ed08322d17e9717ddf652c42df92f0d9b0c52cc5a242c96bc845144" +checksum = "1719b022df1b19866b3021ca8c4c1acf860b853f5fc17388e91c8f41064974dc" dependencies = [ "once_cell", "rayon", @@ -4832,9 +4850,9 @@ dependencies = [ [[package]] name = "simd-json" -version = "0.7.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3375b6c3d8c048ba09c8b4b6c3f1d3f35e06b71db07d231c323943a949e1b8" +checksum = "dbb0157804d962d743f114497a9188fa8b6f9a201faf77ce49454d9e0e61b316" dependencies = [ "halfbrown", "lexical-core", @@ -4931,6 +4949,15 @@ dependencies = [ "lock_api", ] +[[package]] +name = "sqlparser" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db67dc6ef36edb658196c3fef0464a80b53dbbc194a904e81f9bd4190f9ecc5b" +dependencies = [ + "log", +] + [[package]] name = "sqlparser" version = "0.32.0" @@ -5650,9 +5677,9 @@ dependencies = [ [[package]] name = "value-trait" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "995de1aa349a0dc50f4aa40870dce12961a30229027230bad09acd2843edbe9e" +checksum = "733e1909354ceff16f7e1f7ddebc063455b8fda4968d91760a7c6115b96647a5" dependencies = [ "float-cmp", "halfbrown", diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index b60d0cda82..7d1744b57e 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -128,7 +128,7 @@ features = [ "checked_arithmetic", "concat_str", "cross_join", - "csv-file", + "csv", "cum_agg", "default", "dtype-categorical", @@ -151,7 +151,7 @@ features = [ "to_dummies", ] optional = true -version = "0.28.0" +version = "0.29.0" [target.'cfg(windows)'.dependencies.windows] features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_SystemServices"] diff --git a/crates/nu-command/src/dataframe/eager/sql_context.rs b/crates/nu-command/src/dataframe/eager/sql_context.rs index 1dd0bc7815..5aed29f183 100644 --- a/crates/nu-command/src/dataframe/eager/sql_context.rs +++ b/crates/nu-command/src/dataframe/eager/sql_context.rs @@ -163,7 +163,7 @@ impl SQLContext { .collect() .unwrap_or_default() .schema() - .get_index(shm_p) + .get_at_index(shm_p) .unwrap_or((&"".into(), &DataType::Null)) .0) }) diff --git a/crates/nu-command/src/dataframe/expressions/expressions_macro.rs b/crates/nu-command/src/dataframe/expressions/expressions_macro.rs index 06681f8bd3..fcb4695773 100644 --- a/crates/nu-command/src/dataframe/expressions/expressions_macro.rs +++ b/crates/nu-command/src/dataframe/expressions/expressions_macro.rs @@ -134,15 +134,15 @@ macro_rules! expr_command { // Expands to a command definition for a list expression expr_command!( ExprList, - "dfr list", + "dfr implode", "Aggregates a group to a Series", vec![Example { description: "", example: "", result: None, }], - list, - test_list + implode, + test_implode ); // ExprAggGroups command diff --git a/crates/nu-command/src/dataframe/lazy/aggregate.rs b/crates/nu-command/src/dataframe/lazy/aggregate.rs index 3f7c7f625c..44e005df8c 100644 --- a/crates/nu-command/src/dataframe/lazy/aggregate.rs +++ b/crates/nu-command/src/dataframe/lazy/aggregate.rs @@ -160,7 +160,7 @@ fn get_col_name(expr: &Expr) -> Option { | polars::prelude::AggExpr::First(e) | polars::prelude::AggExpr::Last(e) | polars::prelude::AggExpr::Mean(e) - | polars::prelude::AggExpr::List(e) + | polars::prelude::AggExpr::Implode(e) | polars::prelude::AggExpr::Count(e) | polars::prelude::AggExpr::Sum(e) | polars::prelude::AggExpr::AggGroups(e) @@ -170,6 +170,7 @@ fn get_col_name(expr: &Expr) -> Option { }, Expr::Filter { input: expr, .. } | Expr::Slice { input: expr, .. } + | Expr::Cache { input: expr, .. } | Expr::Cast { expr, .. } | Expr::Sort { expr, .. } | Expr::Take { expr, .. } diff --git a/crates/nu-command/src/dataframe/values/nu_expression/mod.rs b/crates/nu-command/src/dataframe/values/nu_expression/mod.rs index 3dc1237602..83f931bcb1 100644 --- a/crates/nu-command/src/dataframe/values/nu_expression/mod.rs +++ b/crates/nu-command/src/dataframe/values/nu_expression/mod.rs @@ -277,7 +277,7 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { | AggExpr::First(expr) | AggExpr::Last(expr) | AggExpr::Mean(expr) - | AggExpr::List(expr) + | AggExpr::Implode(expr) | AggExpr::Count(expr) | AggExpr::Sum(expr) | AggExpr::AggGroups(expr) @@ -578,6 +578,21 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { span, } } + Expr::Cache { input, id } => { + let input = expr_to_value(input.as_ref(), span); + let id = Value::String { + val: format!("{id:?}"), + span, + }; + + let cols = vec!["input".into(), "id".into()]; + + Value::Record { + cols, + vals: vec![input, id], + span, + } + } Expr::Window { function, partition_by,