mirror of
https://github.com/nushell/nushell
synced 2024-11-10 07:04:13 +00:00
Update to polars 0.16 (#4013)
* update to polars 0.16 * enabled features for polars
This commit is contained in:
Fernando Herrera
GitHub
parent
cc3653cfd9
commit
c164ef5489
6 changed files with 172 additions and 109 deletions
251
Cargo.lock
generated
251
Cargo.lock
generated
|
|
@ -144,26 +144,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
|
||||
|
||||
[[package]]
|
||||
name = "arrow"
|
||||
version = "5.2.0"
|
||||
name = "arrow2"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31ef7f4383e52f2da72da147040148a385b7cbf9ec0fc45cd3e2f369a9d442fe"
|
||||
checksum = "bd13b40184904cea33bbc7388c394a995ae5e74d21314c5084d6392753415510"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"base64",
|
||||
"chrono",
|
||||
"csv",
|
||||
"flatbuffers",
|
||||
"hex 0.4.3",
|
||||
"futures 0.3.16",
|
||||
"hash_hasher",
|
||||
"indexmap",
|
||||
"lazy_static",
|
||||
"lexical-core",
|
||||
"multiversion",
|
||||
"num 0.4.0",
|
||||
"rand 0.8.4",
|
||||
"num-traits",
|
||||
"parquet2",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"streaming-iterator",
|
||||
"strength_reduce",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -195,6 +197,27 @@ version = "1.10.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8056f1455169ab86dd47b47391e4ab0cbd25410a70e9fe675544f49bafaf952"
|
||||
|
||||
[[package]]
|
||||
name = "async-stream"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625"
|
||||
dependencies = [
|
||||
"async-stream-impl",
|
||||
"futures-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-stream-impl"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
version = "0.1.51"
|
||||
|
|
@ -317,6 +340,15 @@ version = "1.2.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
|
||||
|
||||
[[package]]
|
||||
name = "bitpacking"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8c7d2ac73c167c06af4a5f37e6e59d84148d57ccbe4480b76f0273eefea82d7"
|
||||
dependencies = [
|
||||
"crunchy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blake2b_simd"
|
||||
version = "0.5.11"
|
||||
|
|
@ -773,6 +805,12 @@ dependencies = [
|
|||
"winapi 0.3.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crunchy"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-mac"
|
||||
version = "0.11.1"
|
||||
|
|
@ -1216,12 +1254,6 @@ dependencies = [
|
|||
"regex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fast-float"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c"
|
||||
|
||||
[[package]]
|
||||
name = "fd-lock"
|
||||
version = "3.0.0"
|
||||
|
|
@ -1640,6 +1672,12 @@ dependencies = [
|
|||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hash_hasher"
|
||||
version = "2.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.11.2"
|
||||
|
|
@ -1907,9 +1945,13 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "integer-encoding"
|
||||
version = "1.1.7"
|
||||
version = "3.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f"
|
||||
checksum = "90c11140ffea82edce8dcd74137ce9324ec24b3cf0175fc9d7e29164da9915b8"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iovec"
|
||||
|
|
@ -1988,24 +2030,74 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
|||
|
||||
[[package]]
|
||||
name = "lexical"
|
||||
version = "5.2.2"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f404a90a744e32e8be729034fc33b90cf2a56418fbf594d69aa3c0214ad414e5"
|
||||
checksum = "23f26223deed22acc21a8aea097679f86af968272e29c4157696cdd8c983bf91"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"lexical-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lexical-core"
|
||||
version = "0.7.6"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
|
||||
checksum = "d32c80337884d5044fe54e9c1b8d64b92de67e10d9312e472a8ff6d6ea849daf"
|
||||
dependencies = [
|
||||
"arrayvec 0.5.2",
|
||||
"bitflags",
|
||||
"cfg-if 1.0.0",
|
||||
"ryu",
|
||||
"lexical-parse-float",
|
||||
"lexical-parse-integer",
|
||||
"lexical-util",
|
||||
"lexical-write-float",
|
||||
"lexical-write-integer",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lexical-parse-float"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "673a01c82cb851a33bb46cacc44c3ad5e7b39ea3b8d22ade21646221df58e45f"
|
||||
dependencies = [
|
||||
"lexical-parse-integer",
|
||||
"lexical-util",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lexical-parse-integer"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2c92badda8cc0fc4f3d3cc1c30aaefafb830510c8781ce4e8669881f3ed53ac"
|
||||
dependencies = [
|
||||
"lexical-util",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lexical-util"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ff669ccaae16ee33af90dc51125755efed17f1309626ba5c12052512b11e291"
|
||||
dependencies = [
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lexical-write-float"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f93601479eae2b41ad465e1f813ea98780069ef1d69063145e76c1bd108ab769"
|
||||
dependencies = [
|
||||
"lexical-util",
|
||||
"lexical-write-integer",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lexical-write-integer"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ece956492e0e40fd95ef8658a34d53a3b8c2015762fdcaaff2167b28de1f56ef"
|
||||
dependencies = [
|
||||
"lexical-util",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
|
|
@ -3431,33 +3523,34 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "parquet"
|
||||
version = "5.2.0"
|
||||
name = "parquet-format-async-temp"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e29eb4b97c8b1d573437e71a9b6f4e8c7c8fab7da62fed3acc6361e3f1fa574c"
|
||||
checksum = "03abc2f9c83fe9ceec83f47c76cc071bfd56caba33794340330f35623ab1f544"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"base64",
|
||||
"brotli",
|
||||
"async-trait",
|
||||
"byteorder",
|
||||
"chrono",
|
||||
"flate2",
|
||||
"lz4",
|
||||
"num-bigint 0.4.1",
|
||||
"parquet-format",
|
||||
"rand 0.8.4",
|
||||
"snap",
|
||||
"thrift",
|
||||
"zstd",
|
||||
"futures 0.3.16",
|
||||
"integer-encoding",
|
||||
"ordered-float",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parquet-format"
|
||||
version = "2.6.1"
|
||||
name = "parquet2"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5bc6b23543b5dedc8f6cce50758a35e5582e148e0cfa26bd0cacd569cda5b71"
|
||||
checksum = "73758f76c8c842dfd846dfbc6ee05779aff626908531fffc3305a30cbccc55e7"
|
||||
dependencies = [
|
||||
"thrift",
|
||||
"async-stream",
|
||||
"bitpacking",
|
||||
"brotli",
|
||||
"flate2",
|
||||
"futures 0.3.16",
|
||||
"lz4",
|
||||
"parquet-format-async-temp",
|
||||
"snap",
|
||||
"streaming-iterator",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3637,9 +3730,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "polars"
|
||||
version = "0.15.1"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "080a75a211a2b0feb01c85efbdb3906593150629679f2b1147ae80c98cde87ea"
|
||||
checksum = "e3545b707f5dc6ac936540da732dbeb51f7f3a1c52bba7561be75fba07c38ee8"
|
||||
dependencies = [
|
||||
"polars-core",
|
||||
"polars-io",
|
||||
|
|
@ -3648,31 +3741,30 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "polars-arrow"
|
||||
version = "0.15.1"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ff08a43b4d19710717e35d91b036bb636fdf7daa794c9ccc38e7ba95aca1135"
|
||||
checksum = "4bebe1310113f2330e92faf7c67634b69a3c30fede04fe5236a258afb43d9d5f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow2",
|
||||
"num 0.4.0",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polars-core"
|
||||
version = "0.15.1"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4929a06fa295e6f702a7d516a84f5cf65cfb72c675daf3cf99031a0d1ef6fe6f"
|
||||
checksum = "8da53d641dcaebe2a3b6cc442f657fd83709cb22a67677eabeb24cb004ebd04e"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
"arrow",
|
||||
"arrow2",
|
||||
"chrono",
|
||||
"hashbrown",
|
||||
"itertools",
|
||||
"lazy_static",
|
||||
"num 0.4.0",
|
||||
"num_cpus",
|
||||
"parquet",
|
||||
"polars-arrow",
|
||||
"prettytable-rs",
|
||||
"rand 0.7.3",
|
||||
|
|
@ -3687,34 +3779,31 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "polars-io"
|
||||
version = "0.15.1"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "422d2405b409defef53de50d2eaf3ab77080c95505d191722c6236d9f3a022a7"
|
||||
checksum = "708e3bf4d8866e4c8a2defe36d7a5359d69e3884a9da0a9cc1f2e2eac194177b"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
"arrow",
|
||||
"arrow2",
|
||||
"csv-core",
|
||||
"dirs 3.0.2",
|
||||
"fast-float",
|
||||
"lazy_static",
|
||||
"lexical",
|
||||
"memmap2",
|
||||
"num 0.4.0",
|
||||
"num_cpus",
|
||||
"parquet",
|
||||
"polars-arrow",
|
||||
"polars-core",
|
||||
"rayon",
|
||||
"regex",
|
||||
"simdutf8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polars-lazy"
|
||||
version = "0.15.1"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50a6302df2cf8f261823fefc032b5fc7c6f8d2eb5c55c9bf305d472d1f500376"
|
||||
checksum = "9d1ffc1bb44a8ddd0607448f49027d69debe3f7243aad3b17f713780d0d47205"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"itertools",
|
||||
|
|
@ -4660,12 +4749,6 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simdutf8"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c970da16e7c682fa90a261cf0724dee241c9f7831635ecc4e988ae8f3b505559"
|
||||
|
||||
[[package]]
|
||||
name = "similar"
|
||||
version = "1.3.0"
|
||||
|
|
@ -4741,6 +4824,18 @@ version = "1.0.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d44a3643b4ff9caf57abcee9c2c621d6c03d9135e0d8b589bd9afb5992cb176a"
|
||||
|
||||
[[package]]
|
||||
name = "streaming-iterator"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "303235c177994a476226b80d076bd333b7b560fb05bd242a10609d11b07f81f5"
|
||||
|
||||
[[package]]
|
||||
name = "strength_reduce"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3ff2f71c82567c565ba4b3009a9350a96a7269eaa4001ebedae926230bc2254"
|
||||
|
||||
[[package]]
|
||||
name = "string_cache"
|
||||
version = "0.8.1"
|
||||
|
|
@ -5004,28 +5099,6 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "threadpool"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
|
||||
dependencies = [
|
||||
"num_cpus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thrift"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c6d965454947cc7266d22716ebfd07b18d84ebaf35eec558586bbb2a8cb6b5b"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"integer-encoding",
|
||||
"log",
|
||||
"ordered-float",
|
||||
"threadpool",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiff"
|
||||
version = "0.6.1"
|
||||
|
|
|
|||
|
|
@ -104,9 +104,9 @@ zip = { version="0.5.9", optional=true }
|
|||
digest = "0.9.0"
|
||||
|
||||
[dependencies.polars]
|
||||
version = "0.15.1"
|
||||
version = "0.16.0"
|
||||
optional = true
|
||||
features = ["parquet", "json", "random", "pivot", "strings", "is_in", "temporal"]
|
||||
features = ["parquet", "json", "random", "pivot", "strings", "is_in", "temporal", "cum_agg", "rolling_window"]
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
umask = "1.0.0"
|
||||
|
|
@ -141,4 +141,4 @@ dataframe = ["nu-protocol/dataframe", "polars"]
|
|||
fetch = ["reqwest", "tokio"]
|
||||
post = ["reqwest", "tokio"]
|
||||
sys = ["sysinfo"]
|
||||
ps = ["sysinfo"]
|
||||
ps = ["sysinfo"]
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use nu_protocol::{
|
|||
};
|
||||
|
||||
use nu_source::Tagged;
|
||||
use polars::prelude::{CsvEncoding, CsvReader, JsonReader, ParquetReader, PolarsError, SerReader};
|
||||
use polars::prelude::{CsvEncoding, CsvReader, JsonReader, ParquetReader, SerReader};
|
||||
use std::fs::File;
|
||||
|
||||
pub struct DataFrame;
|
||||
|
|
@ -206,15 +206,6 @@ fn from_csv(args: CommandArgs) -> Result<polars::prelude::DataFrame, ShellError>
|
|||
|
||||
match csv_reader.finish() {
|
||||
Ok(df) => Ok(df),
|
||||
Err(e) => match e {
|
||||
PolarsError::Other(_) => Err(ShellError::labeled_error_with_secondary(
|
||||
"Schema error",
|
||||
"Error with the inferred schema",
|
||||
&file.tag.span,
|
||||
"You can use the argument 'infer_schema' with a number of rows large enough to better infer the schema",
|
||||
&file.tag.span,
|
||||
)),
|
||||
_ => Err(parse_polars_error::<&str>(&e, &file.tag.span, None)),
|
||||
},
|
||||
Err(e) => Err(parse_polars_error::<&str>(&e, &file.tag.span, None)),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -101,9 +101,9 @@ fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
|
|||
|
||||
let cum_type = CumType::from_str(&cum_type.item, &cum_type.tag.span)?;
|
||||
let mut res = match cum_type {
|
||||
CumType::Max => series.cum_max(reverse),
|
||||
CumType::Min => series.cum_min(reverse),
|
||||
CumType::Sum => series.cum_sum(reverse),
|
||||
CumType::Max => series.cummax(reverse),
|
||||
CumType::Min => series.cummin(reverse),
|
||||
CumType::Sum => series.cumsum(reverse),
|
||||
};
|
||||
|
||||
let name = format!("{}_{}", series.name(), cum_type.to_str());
|
||||
|
|
|
|||
|
|
@ -53,13 +53,12 @@ pub(crate) fn parse_polars_error<T: AsRef<str>>(
|
|||
PolarsError::DataTypeMisMatch(_) => "Data Type Mismatch",
|
||||
PolarsError::NotFound(_) => "Not Found",
|
||||
PolarsError::ShapeMisMatch(_) => "Shape Mismatch",
|
||||
PolarsError::Other(_) => "Other",
|
||||
PolarsError::ComputeError(_) => "Computer error",
|
||||
PolarsError::OutOfBounds(_) => "Out Of Bounds",
|
||||
PolarsError::NoSlice => "No Slice",
|
||||
PolarsError::NoData(_) => "No Data",
|
||||
PolarsError::ValueError(_) => "Value Error",
|
||||
PolarsError::MemoryNotAligned => "Memory Not Aligned",
|
||||
PolarsError::ParquetError(_) => "Parquet Error",
|
||||
PolarsError::RandError(_) => "Rand Error",
|
||||
PolarsError::HasNullValues(_) => "Has Null Values",
|
||||
PolarsError::UnknownSchema(_) => "Unknown Schema",
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ serde_yaml = "0.8.16"
|
|||
toml = "0.5.8"
|
||||
|
||||
[dependencies.polars]
|
||||
version = "0.15.1"
|
||||
version = "0.16.0"
|
||||
optional = true
|
||||
features = ["default", "serde", "rows", "strings", "checked_arithmetic", "object", "dtype-duration-ns"]
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue