Update polars to 0.28 (#9136)

# Description
Update polars to 0.28.
Luckily, it didn't require major changes.

# User-Facing Changes
None.
(Apart from the fact that certain error messages will stop breaking
table formatting)
This commit is contained in:
Hofer-Julian 2023-05-08 17:42:53 +02:00 committed by GitHub
parent 388e84e7ef
commit d5ae979094
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 142 additions and 135 deletions

225
Cargo.lock generated
View file

@ -36,7 +36,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
dependencies = [
"cfg-if 1.0.0",
"const-random",
"getrandom 0.2.9",
"once_cell",
"version_check",
@ -112,12 +111,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d"
[[package]]
name = "anyhow"
version = "1.0.70"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4"
[[package]]
name = "array-init-cursor"
version = "0.2.0"
@ -148,13 +141,13 @@ dependencies = [
[[package]]
name = "arrow2"
version = "0.16.0"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a4c5b03335bc1cb0fd9f5297f8fd3bbfd6fb04f3cb0bc7d6c91b7128cb8336a"
checksum = "a0f73029049896b3d70ba17756afef171ceef3569016cfa9dbca58d29e0e16f9"
dependencies = [
"ahash 0.8.3",
"arrow-format",
"base64 0.13.1",
"base64 0.21.0",
"bytemuck",
"chrono",
"dyn-clone",
@ -176,7 +169,7 @@ dependencies = [
"simdutf8",
"streaming-iterator",
"strength_reduce",
"zstd 0.11.2+zstd.1.5.2",
"zstd",
]
[[package]]
@ -237,6 +230,15 @@ dependencies = [
"syn 2.0.15",
]
[[package]]
name = "atoi"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528"
dependencies = [
"num-traits",
]
[[package]]
name = "atomic-polyfill"
version = "0.1.11"
@ -665,28 +667,6 @@ dependencies = [
"windows-sys 0.42.0",
]
[[package]]
name = "const-random"
version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e"
dependencies = [
"const-random-macro",
"proc-macro-hack",
]
[[package]]
name = "const-random-macro"
version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb"
dependencies = [
"getrandom 0.2.9",
"once_cell",
"proc-macro-hack",
"tiny-keccak",
]
[[package]]
name = "const_format"
version = "0.2.30"
@ -874,12 +854,6 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "crunchy"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
[[package]]
name = "crypto-common"
version = "0.1.6"
@ -1284,6 +1258,12 @@ dependencies = [
"regex",
]
[[package]]
name = "fast-float"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c"
[[package]]
name = "fastrand"
version = "1.9.0"
@ -2560,22 +2540,24 @@ dependencies = [
[[package]]
name = "multiversion"
version = "0.6.1"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "025c962a3dd3cc5e0e520aa9c612201d127dcdf28616974961a649dca64f5373"
checksum = "e6a87eede2251ca235e5573086d01d2ab6b59dfaea54c2be10f9320980f7e8f7"
dependencies = [
"multiversion-macros",
"target-features",
]
[[package]]
name = "multiversion-macros"
version = "0.6.1"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8a3e2bde382ebf960c1f3e79689fa5941625fe9bf694a1cb64af3e85faff3af"
checksum = "1af1abf82261d780d114014eff4b555e47d823f3b84f893c4388572b40e089fb"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
"target-features",
]
[[package]]
@ -2772,7 +2754,7 @@ dependencies = [
"percent-encoding",
"reedline",
"rstest",
"sysinfo 0.28.4",
"sysinfo",
"thiserror",
"unicode-segmentation",
]
@ -2892,7 +2874,7 @@ dependencies = [
"serde_yaml",
"sha2",
"sqlparser",
"sysinfo 0.28.4",
"sysinfo",
"tabled",
"terminal_size 0.2.6",
"thiserror",
@ -2922,7 +2904,7 @@ dependencies = [
"nu-protocol",
"nu-utils",
"serde",
"sysinfo 0.28.4",
"sysinfo",
]
[[package]]
@ -3546,7 +3528,7 @@ dependencies = [
"seq-macro",
"snap",
"streaming-decompression",
"zstd 0.12.3+zstd.1.5.2",
"zstd",
]
[[package]]
@ -3775,9 +3757,9 @@ dependencies = [
[[package]]
name = "polars"
version = "0.27.2"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8918f4add49e6244bae2fe91cac89be339f2d6a77c59a4df3d7348bd40f98d1e"
checksum = "b8f04742abbadbd348d73a803617e6df61464485a410361e9c6634e55a4bc01c"
dependencies = [
"getrandom 0.2.9",
"polars-core",
@ -3790,34 +3772,38 @@ dependencies = [
[[package]]
name = "polars-arrow"
version = "0.27.2"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06e57a7b929edf6c73475dbc3f63d35152f14f4a9455476acc6127d770daa0f6"
checksum = "eacd179fe0cc7cf18e12a80cfda3e6d15ce907655203cc69277e883a925f9bd8"
dependencies = [
"arrow2",
"hashbrown 0.13.2",
"num 0.4.0",
"multiversion",
"num-traits",
"polars-error",
"serde",
"thiserror",
]
[[package]]
name = "polars-core"
version = "0.27.2"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a440cd53916f1a87fac1fda36cd7cc2d226247b4d4570d96242da5fa7f07b2a"
checksum = "04145d95791f31a0df3b2d3d600cbb5d0d43ccc105d74e871ffe7351a5e70cbe"
dependencies = [
"ahash 0.8.3",
"anyhow",
"arrow2",
"bitflags",
"chrono",
"comfy-table",
"either",
"hashbrown 0.13.2",
"indexmap",
"num 0.4.0",
"num-traits",
"once_cell",
"polars-arrow",
"polars-error",
"polars-row",
"polars-utils",
"rand 0.8.5",
"rand_distr",
@ -3832,25 +3818,39 @@ dependencies = [
]
[[package]]
name = "polars-io"
version = "0.27.2"
name = "polars-error"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d941750cba70a3acf150b959fcf446c09e8a8a4a35d03472d941bd6740bc43"
checksum = "5fc2fb4ff74f7224b4096b3551b309c50bfd49bb0b66fb3948e70feffb1a8e51"
dependencies = [
"arrow2",
"regex",
"thiserror",
]
[[package]]
name = "polars-io"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4d186157d427b39085f812e74eefb04608b7bd033ef3a7f848b93a5939c396d"
dependencies = [
"ahash 0.8.3",
"anyhow",
"arrow2",
"async-trait",
"bytes",
"chrono",
"dirs",
"fast-float",
"futures",
"lexical",
"lexical-core",
"memchr",
"memmap2",
"num 0.4.0",
"num-traits",
"once_cell",
"polars-arrow",
"polars-core",
"polars-error",
"polars-time",
"polars-utils",
"rayon",
@ -3859,13 +3859,14 @@ dependencies = [
"serde_json",
"simd-json",
"simdutf8",
"tokio",
]
[[package]]
name = "polars-lazy"
version = "0.27.2"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d227fcb817485be462748d3172d2e55c61d56fbdc7fd56c24b72fa2e510e7be6"
checksum = "385b1beb0ae4f1c8fbe6aef9769b8e0a22a8b56ba494e382f17de82ce653702f"
dependencies = [
"ahash 0.8.3",
"bitflags",
@ -3879,36 +3880,41 @@ dependencies = [
"polars-time",
"polars-utils",
"rayon",
"smartstring",
]
[[package]]
name = "polars-ops"
version = "0.27.2"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36638340fd9f4377dab11f307877ebb5bdac3bc9b25ea32a771584de76e5280a"
checksum = "b8beae89044ad6f5a2237103852d2650c7a52e7b339e356c1d707b9427b48ba7"
dependencies = [
"arrow2",
"either",
"memchr",
"polars-arrow",
"polars-core",
"polars-utils",
"serde",
"smartstring",
]
[[package]]
name = "polars-pipe"
version = "0.27.2"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac1dfae18a14a812119d3328dae8790be53f1bc0fbcb977606dcf0f181490a8f"
checksum = "59c7fbda6ff77db1ae1a0d8f210b9729b2c0cb7c584345504a8282e3bd38e97d"
dependencies = [
"crossbeam-channel",
"enum_dispatch",
"hashbrown 0.13.2",
"num 0.4.0",
"num-traits",
"polars-arrow",
"polars-core",
"polars-io",
"polars-ops",
"polars-plan",
"polars-row",
"polars-utils",
"rayon",
"smartstring",
@ -3916,11 +3922,12 @@ dependencies = [
[[package]]
name = "polars-plan"
version = "0.27.2"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ca57df4974f25fa0642ae18ef00c0836c5120d4371be77a34d4684173b043c3"
checksum = "11b788cb03c651c9b92c2293b0adc07cb89ae64d7f187310cb7d62d688ccd9d8"
dependencies = [
"ahash 0.8.3",
"arrow2",
"once_cell",
"polars-arrow",
"polars-core",
@ -3930,16 +3937,29 @@ dependencies = [
"polars-utils",
"rayon",
"serde",
"smartstring",
]
[[package]]
name = "polars-row"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bfcd216b7953321a6946c38d352fdfa67260a4d2125cbb4a1c3b46f03d0bebb"
dependencies = [
"arrow2",
"polars-error",
"polars-utils",
]
[[package]]
name = "polars-time"
version = "0.27.2"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d961a9ccbe3c739de063fbf78449b3c172baf3379f958769c42ee9c309289786"
checksum = "5384945cc104ba99683455760dcfe1048e4025bfaf78043e14450cc8c988a745"
dependencies = [
"arrow2",
"atoi",
"chrono",
"lexical",
"now",
"once_cell",
"polars-arrow",
@ -3948,17 +3968,19 @@ dependencies = [
"polars-utils",
"regex",
"serde",
"smartstring",
]
[[package]]
name = "polars-utils"
version = "0.27.2"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a741a3325c544c97c7a9ff57d857f089b60041bd92b06c41582df6940ffaa05b"
checksum = "7c1823141ed08322d17e9717ddf652c42df92f0d9b0c52cc5a242c96bc845144"
dependencies = [
"once_cell",
"rayon",
"sysinfo 0.27.8",
"smartstring",
"sysinfo",
]
[[package]]
@ -4873,6 +4895,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
dependencies = [
"autocfg",
"serde",
"static_assertions",
"version_check",
]
@ -5086,20 +5109,6 @@ dependencies = [
"windows-sys 0.45.0",
]
[[package]]
name = "sysinfo"
version = "0.27.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a902e9050fca0a5d6877550b769abd2bd1ce8c04634b941dbe2809735e1a1e33"
dependencies = [
"cfg-if 1.0.0",
"core-foundation-sys",
"libc",
"ntapi",
"once_cell",
"winapi 0.3.9",
]
[[package]]
name = "sysinfo"
version = "0.28.4"
@ -5141,6 +5150,12 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "target-features"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24840de800c1707d75c800893dbd727a5e1501ce921944e602f0698167491e36"
[[package]]
name = "tempfile"
version = "3.5.0"
@ -5287,15 +5302,6 @@ dependencies = [
"time-core",
]
[[package]]
name = "tiny-keccak"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
dependencies = [
"crunchy",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
@ -6131,32 +6137,13 @@ dependencies = [
"flate2",
]
[[package]]
name = "zstd"
version = "0.11.2+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4"
dependencies = [
"zstd-safe 5.0.2+zstd.1.5.2",
]
[[package]]
name = "zstd"
version = "0.12.3+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806"
dependencies = [
"zstd-safe 6.0.5+zstd.1.5.4",
]
[[package]]
name = "zstd-safe"
version = "5.0.2+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db"
dependencies = [
"libc",
"zstd-sys",
"zstd-safe",
]
[[package]]

View file

@ -151,7 +151,7 @@ features = [
"to_dummies",
]
optional = true
version = "0.27.2"
version = "0.28.0"
[target.'cfg(windows)'.dependencies.windows]
features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_SystemServices"]

View file

@ -97,7 +97,7 @@ fn command(
};
df.as_ref()
.unique(subset_slice, keep_strategy)
.unique(subset_slice, keep_strategy, None)
.map_err(|e| {
ShellError::GenericError(
"Error dropping duplicates".into(),

View file

@ -9,8 +9,8 @@ use nu_protocol::{
use std::{fs::File, io::BufReader, path::PathBuf};
use polars::prelude::{
CsvEncoding, CsvReader, IpcReader, JsonReader, LazyCsvReader, LazyFrame, ParallelStrategy,
ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader,
CsvEncoding, CsvReader, IpcReader, JsonReader, LazyCsvReader, LazyFileListReader, LazyFrame,
ParallelStrategy, ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader,
};
#[derive(Clone)]
@ -146,6 +146,7 @@ fn from_parquet(
row_count: None,
low_memory: false,
cloud_options: None,
use_statistics: false,
};
let df: NuLazyFrame = LazyFrame::scan_parquet(file, args)

View file

@ -164,7 +164,7 @@ impl SQLContext {
.unwrap_or_default()
.schema()
.get_index(shm_p)
.unwrap_or((&"".to_string(), &DataType::Null))
.unwrap_or((&"".into(), &DataType::Null))
.0)
})
.collect::<Vec<_>>();

View file

@ -36,7 +36,7 @@ impl Command for ExprLit {
example: "dfr lit 2 | dfr into-nu",
result: Some(Value::Record {
cols: vec!["expr".into(), "value".into()],
vals: vec![Value::test_string("literal"), Value::test_string("2i64")],
vals: vec![Value::test_string("literal"), Value::test_string("2")],
span: Span::test_data(),
}),
}]

View file

@ -98,7 +98,10 @@ fn command(
multithreaded: true,
};
let mut res = df.as_series(call.head)?.argsort(sort_options).into_series();
let mut res = df
.as_series(call.head)?
.arg_sort(sort_options)
.into_series();
res.rename("arg_sort");
NuDataFrame::try_from_series(vec![res], call.head)

View file

@ -146,6 +146,7 @@ fn command(
by: None,
closed_window: None,
tu: None,
tz: None,
};
let res = match roll_type {
RollType::Max => series.rolling_max(rolling_opts),

View file

@ -82,7 +82,7 @@ fn command(
)
})?;
let res = chunked.contains(&pattern).map_err(|e| {
let res = chunked.contains(&pattern, false).map_err(|e| {
ShellError::GenericError(
"Error searching in series".into(),
e.to_string(),

View file

@ -80,7 +80,18 @@ fn command(
)
})?;
let res = casted.strftime(&fmt).into_series();
let res = casted
.strftime(&fmt)
.map_err(|e| {
ShellError::GenericError(
"Error formatting datetime".into(),
e.to_string(),
Some(call.head),
None,
Vec::new(),
)
})?
.into_series();
NuDataFrame::try_from_series(vec![res.into_series()], call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))

View file

@ -736,7 +736,7 @@ fn contains_series_pat(series: &Series, pat: &str, span: Span) -> Result<Value,
let casted = series.utf8();
match casted {
Ok(casted) => {
let res = casted.contains(pat);
let res = casted.contains(pat, false);
match res {
Ok(res) => {

View file

@ -426,25 +426,29 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value {
span,
}
}
Expr::SortBy { expr, by, reverse } => {
Expr::SortBy {
expr,
by,
descending,
} => {
let expr = expr_to_value(expr.as_ref(), span);
let by: Vec<Value> = by.iter().map(|b| expr_to_value(b, span)).collect();
let by = Value::List { vals: by, span };
let reverse: Vec<Value> = reverse
let descending: Vec<Value> = descending
.iter()
.map(|r| Value::Bool { val: *r, span })
.collect();
let reverse = Value::List {
vals: reverse,
let descending = Value::List {
vals: descending,
span,
};
let cols = vec!["expr".into(), "by".into(), "reverse".into()];
let cols = vec!["expr".into(), "by".into(), "descending".into()];
Value::Record {
cols,
vals: vec![expr, by, reverse],
vals: vec![expr, by, descending],
span,
}
}