Mirrors/nushell
2
0
Fork 0
mirror of https://github.com/nushell/nushell synced 2025-01-14 14:14:13 +00:00
Code Issues Projects Releases Packages Wiki Activity

Update polars to 0.33 (#10672)

Browse source 
# Description
Open question:

Undocumented behavior for the new argument `ambiguous` to the
`as_datetime`
methods. I cheated by passing a default (assuming empty string).
This appears like an API primarily serving the python impl:


https://pola-rs.github.io/polars/py-polars/html/reference/expressions/api/polars.Expr.str.to_datetime.html#polars-expr-str-to-datetime


# User-Facing Changes
Only dependent on breaking changes to the behavior of polars.

# Tests + Formatting
No observed changes to tests

Manually checked `dfr as-datetime`, doesn't seem to panic.
This commit is contained in:
Stefan Holderbach 2023-10-11 21:28:18 +02:00 committed by GitHub
parent c5545c59c6
commit c925537c48
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 59 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

91
Cargo.lock generated
View file

@ -208,9 +208,9 @@ dependencies = [
[[package]] [[package]]
name = "arrow2" name = "arrow2"
version = "0.17.4" version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59c468daea140b747d781a1da9f7db5f0a8e6636d4af20cc539e43d05b0604fa" checksum = "963fef509b757bcbbf9e5ffa23bcb345614d99f4f6f531f97417b27b8604d389"
dependencies = [ dependencies = [
"ahash", "ahash",
"arrow-format", "arrow-format",
@ -226,13 +226,14 @@ dependencies = [
"futures", "futures",
"getrandom", "getrandom",
"hash_hasher", "hash_hasher",
"hashbrown 0.14.1",
"lexical-core", "lexical-core",
"lz4", "lz4",
"multiversion", "multiversion",
"num-traits", "num-traits",
"parquet2", "parquet2",
"regex", "regex",
"regex-syntax 0.6.29", "regex-syntax",
"rustc_version", "rustc_version",
"simdutf8", "simdutf8",
"streaming-iterator", "streaming-iterator",
@ -904,6 +905,16 @@ dependencies = [
"scopeguard", "scopeguard",
] ]
[[package]]
name = "crossbeam-queue"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add"
dependencies = [
"cfg-if",
"crossbeam-utils",
]
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.8.16" version = "0.8.16"
@ -3684,9 +3695,9 @@ dependencies = [
[[package]] [[package]]
name = "polars" name = "polars"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1362d4a136c0ebacb40d88a37ba361738b222fd8a2ee9340a3d8642f698c52b" checksum = "3030de163b9ff2c9dac9a12dcb9be25cc0f2bc7c8e7cd2e4b2592ebed458ce6a"
dependencies = [ dependencies = [
"getrandom", "getrandom",
"polars-core", "polars-core",
@ -3700,9 +3711,9 @@ dependencies = [
[[package]] [[package]]
name = "polars-arrow" name = "polars-arrow"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f967c901fa5da4ca7f64e813d1268488ba97e9b3004cefc579ff851c197a1138" checksum = "35cd38a64fb389fd990e4efd433a36331c995c981d353bfef83b5de4d87f1828"
dependencies = [ dependencies = [
"arrow2", "arrow2",
"hashbrown 0.14.1", "hashbrown 0.14.1",
@ -3716,9 +3727,9 @@ dependencies = [
[[package]] [[package]]
name = "polars-core" name = "polars-core"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b24f92fc5b167f668ff85ab9607dfa72e2c09664cacef59297ee8601dee60126" checksum = "08367c014c07fa8f141680e024f926cab3a1fe839605a8fcf2223647eb45ca71"
dependencies = [ dependencies = [
"ahash", "ahash",
"arrow2", "arrow2",
@ -3748,9 +3759,9 @@ dependencies = [
[[package]] [[package]]
name = "polars-error" name = "polars-error"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40d09c3a7337e53b38c37b57999038440fa39c6801b9ba48afaecd8e16f7ac0a" checksum = "9b20a09651a299979354945819dc2ce017964b80b916954e9d2ce39002a5f949"
dependencies = [ dependencies = [
"arrow2", "arrow2",
"regex", "regex",
@ -3759,17 +3770,16 @@ dependencies = [
[[package]] [[package]]
name = "polars-io" name = "polars-io"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92cab0df9f2a35702fa5aec99edfaabf9ae8e9cdd0acf69e143ad2d132f34f9c" checksum = "88cf4a89c18a90ac20dfbcdfd19ab50ad4ac5a76fc7bb775d3c28bb738cf1f34"
dependencies = [ dependencies = [
"ahash", "ahash",
"arrow2", "arrow2",
"async-trait",
"bytes", "bytes",
"chrono", "chrono",
"fast-float", "fast-float",
"futures", "flate2",
"home", "home",
"lexical", "lexical",
"lexical-core", "lexical-core",
@ -3789,14 +3799,13 @@ dependencies = [
"serde_json", "serde_json",
"simd-json", "simd-json",
"simdutf8", "simdutf8",
"tokio",
] ]
[[package]] [[package]]
name = "polars-json" name = "polars-json"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e58094557cf6446808c7348dcb797885db61815857f6ea02924b35505566e94" checksum = "d6d5666176d681706aef5a06a57597c83391948b3d958f9fbe9b4cf016527eb8"
dependencies = [ dependencies = [
"ahash", "ahash",
"arrow2", "arrow2",
@ -3812,9 +3821,9 @@ dependencies = [
[[package]] [[package]]
name = "polars-lazy" name = "polars-lazy"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c33762ec2a55e01c9f8776b34db86257c70a0a3b3929bd4eb91a52aacf61456" checksum = "5110eab438848c981cc5f541fbc5b21bb263fd707000b4715233074fb2630fcf"
dependencies = [ dependencies = [
"ahash", "ahash",
"bitflags 2.4.0", "bitflags 2.4.0",
@ -3836,9 +3845,9 @@ dependencies = [
[[package]] [[package]]
name = "polars-ops" name = "polars-ops"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e825575c96302d2daedfc205a0062180033c92c55bcd6aafc4e109d4d8849ed0" checksum = "7740d7bc4c2ca08044f9ef599638e116fdd7d687e80d1974b698e390c6ce4252"
dependencies = [ dependencies = [
"argminmax", "argminmax",
"arrow2", "arrow2",
@ -3848,6 +3857,7 @@ dependencies = [
"polars-arrow", "polars-arrow",
"polars-core", "polars-core",
"polars-utils", "polars-utils",
"regex",
"serde", "serde",
"smartstring", "smartstring",
"version_check", "version_check",
@ -3855,10 +3865,12 @@ dependencies = [
[[package]] [[package]]
name = "polars-pipe" name = "polars-pipe"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f2bc9a12da9ed043fb0cb51dbcb87b365e4845b7ab6399d7a81e838460c6974" checksum = "1f30c5e77c5594ddc958a46fe2e021da2feba9c94e767e1d798bd82ac5a33c3b"
dependencies = [ dependencies = [
"crossbeam-channel",
"crossbeam-queue",
"enum_dispatch", "enum_dispatch",
"hashbrown 0.14.1", "hashbrown 0.14.1",
"num-traits", "num-traits",
@ -3876,9 +3888,9 @@ dependencies = [
[[package]] [[package]]
name = "polars-plan" name = "polars-plan"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb67b014f0295e8e9dbb84404a91d666d477b3bc248a2ed51bc442833b16da35" checksum = "678cbeb730e29e50f0f8d844102d15454fc6113a74c667eab046c0e4a4322a9e"
dependencies = [ dependencies = [
"ahash", "ahash",
"arrow2", "arrow2",
@ -3899,9 +3911,9 @@ dependencies = [
[[package]] [[package]]
name = "polars-row" name = "polars-row"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27f54c1956027bf6301948fb4f2837cf6d6b638d8dd1edf3aaeaa19906a986be" checksum = "c52ef8885b9d13f848839594fbab21ad79fc63f7e11c19cdc2cfe9bb03c313ac"
dependencies = [ dependencies = [
"arrow2", "arrow2",
"polars-error", "polars-error",
@ -3910,9 +3922,9 @@ dependencies = [
[[package]] [[package]]
name = "polars-sql" name = "polars-sql"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbfcb15cf8eebd25ea1724109d0153817cd484c6326290585f0736b4e7fcf2f4" checksum = "4d716855267e3516f722287f68cf10e650e33f7197df83a79e680602471456fc"
dependencies = [ dependencies = [
"polars-arrow", "polars-arrow",
"polars-core", "polars-core",
@ -3925,9 +3937,9 @@ dependencies = [
[[package]] [[package]]
name = "polars-time" name = "polars-time"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53f42d2632f5971c9575041d33cbcfb1f996900c40bbf58bc6eb0a0c5efbecea" checksum = "a2eb75a24f11b55a400b52dc19a2a3e949aaaa46a911f99496de4485b1127063"
dependencies = [ dependencies = [
"arrow2", "arrow2",
"atoi", "atoi",
@ -3945,11 +3957,12 @@ dependencies = [
[[package]] [[package]]
name = "polars-utils" name = "polars-utils"
version = "0.32.1" version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c326708a370d71dc6e11a8f4bbc10a8479e1c314dc048ba73543b815cd0bf339" checksum = "2a4a5e743509096322cad39104d56e329fe2748483a3354a0f0c354724f3cef6"
dependencies = [ dependencies = [
"ahash", "ahash",
"bytemuck",
"hashbrown 0.14.1", "hashbrown 0.14.1",
"num-traits", "num-traits",
"once_cell", "once_cell",
@ -4269,7 +4282,7 @@ dependencies = [
"aho-corasick", "aho-corasick",
"memchr", "memchr",
"regex-automata", "regex-automata",
"regex-syntax 0.7.5", "regex-syntax",
] ]
[[package]] [[package]]
@ -4280,15 +4293,9 @@ checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9"
dependencies = [ dependencies = [
"aho-corasick", "aho-corasick",
"memchr", "memchr",
"regex-syntax 0.7.5", "regex-syntax",
] ]
[[package]]
name = "regex-syntax"
version = "0.6.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]] [[package]]
name = "regex-syntax" name = "regex-syntax"
version = "0.7.5" version = "0.7.5"

6
crates/nu-cmd-dataframe/Cargo.toml
View file

@ -24,7 +24,7 @@ indexmap = { version = "2.0" }
num = { version = "0.4", optional = true } num = { version = "0.4", optional = true }
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
sqlparser = { version = "0.36.1", optional = true } sqlparser = { version = "0.36.1", optional = true }
polars-io = { version = "0.32", features = ["avro"], optional = true } polars-io = { version = "0.33", features = ["avro"], optional = true }
[dependencies.polars] [dependencies.polars]
features = [ features = [
@ -38,7 +38,7 @@ features = [
"dtype-categorical", "dtype-categorical",
"dtype-datetime", "dtype-datetime",
"dtype-struct", "dtype-struct",
"dynamic_groupby", "dynamic_group_by",
"ipc", "ipc",
"is_in", "is_in",
"json", "json",
@ -54,7 +54,7 @@ features = [
"to_dummies", "to_dummies",
] ]
optional = true optional = true
version = "0.32" version = "0.33"
[features] [features]
dataframe = ["num", "polars", "polars-io", "sqlparser"] dataframe = ["num", "polars", "polars-io", "sqlparser"]

2
crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs
View file

@ -147,7 +147,7 @@ impl SQLContext {
.enumerate() .enumerate()
.map(|(agg_pj, (proj_p, expr))| (expr.clone(), (proj_p, agg_pj + group_by.len()))) .map(|(agg_pj, (proj_p, expr))| (expr.clone(), (proj_p, agg_pj + group_by.len())))
.unzip(); .unzip();
let agg_df = df.groupby(group_by).agg(agg_projection); let agg_df = df.group_by(group_by).agg(agg_projection);
let mut final_proj_pos = groupby_pos let mut final_proj_pos = groupby_pos
.into_iter() .into_iter()
.chain(agg_proj_pos) .chain(agg_proj_pos)

2
crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs
View file

@ -131,7 +131,7 @@ impl Command for ToLazyGroupBy {
let group_by = NuLazyGroupBy { let group_by = NuLazyGroupBy {
schema: lazy.schema.clone(), schema: lazy.schema.clone(),
from_eager: lazy.from_eager, from_eager: lazy.from_eager,
group_by: Some(lazy.into_polars().groupby(&expressions)), group_by: Some(lazy.into_polars().group_by(&expressions)),
}; };
Ok(PipelineData::Value(group_by.into_value(call.head), None)) Ok(PipelineData::Value(group_by.into_value(call.head), None))

4
crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs
View file

@ -148,7 +148,7 @@ fn command(
TimeUnit::Nanoseconds, TimeUnit::Nanoseconds,
false, false,
None, None,
None, &Default::default(),
) )
} else { } else {
casted.as_datetime( casted.as_datetime(
@ -157,7 +157,7 @@ fn command(
false, false,
false, false,
None, None,
None, &Default::default(),
) )
}; };

8
crates/nu-cmd-dataframe/src/dataframe/series/masks/is_in.rs
View file

@ -6,7 +6,7 @@ use nu_protocol::{
engine::{Command, EngineState, Stack}, engine::{Command, EngineState, Stack},
Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value, Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
}; };
use polars::prelude::IntoSeries; use polars::prelude::{is_in, IntoSeries};
#[derive(Clone)] #[derive(Clone)]
pub struct IsIn; pub struct IsIn;
@ -71,16 +71,14 @@ fn command(
call: &Call, call: &Call,
input: PipelineData, input: PipelineData,
) -> Result<PipelineData, ShellError> { ) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?; let df = NuDataFrame::try_from_pipeline(input, call.head)?.as_series(call.head)?;
let other_value: Value = call.req(engine_state, stack, 0)?; let other_value: Value = call.req(engine_state, stack, 0)?;
let other_span = other_value.span(); let other_span = other_value.span();
let other_df = NuDataFrame::try_from_value(other_value)?; let other_df = NuDataFrame::try_from_value(other_value)?;
let other = other_df.as_series(other_span)?; let other = other_df.as_series(other_span)?;
let mut res = df let mut res = is_in(&df, &other)
.as_series(call.head)?
.is_in(&other)
.map_err(|e| { .map_err(|e| {
ShellError::GenericError( ShellError::GenericError(
"Error finding in other".into(), "Error finding in other".into(),