From 00b3c2036a0295bc34bf4d96a124621acf3b21f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20N=2E=20Robalino?= Date: Tue, 12 Nov 2019 03:38:55 -0500 Subject: [PATCH] This is part of on-going work with capabilities when working with tables and able to work with them for data processing & viewing purposes. At the moment, certain ways to process said tables we are able to view a histogram of a given column. As usage matures, we may find certain core commands that could be used ergonomically when working with tables on Nu. --- README.md | 2 + src/cli.rs | 4 +- src/commands.rs | 44 ++++++++--------- src/commands/histogram.rs | 97 ++++++++++++++++++++++---------------- src/commands/map_max_by.rs | 40 ++++++++-------- src/commands/t_sort_by.rs | 38 +++++++-------- tests/commands_test.rs | 31 +++++++++++- 7 files changed, 147 insertions(+), 109 deletions(-) diff --git a/README.md b/README.md index 46a4c45ac8..b1ce4feec1 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,7 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat | format pattern | Format table row data as a string following the given pattern | | get column-or-column-path | Open column and get data from the corresponding cells | | group-by column | Creates a new table with the data from the table rows grouped by the column given | +| histogram column ...column-names | Creates a new table with a histogram based on the column name passed in, optionally give the frequency column name | inc (column-or-column-path) | Increment a value or version. Optionally use the column of a table | | insert column-or-column-path value | Insert a new column to the table | | last amount | Show only the last number of rows | @@ -267,6 +268,7 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat | reverse | Reverses the table. | | skip amount | Skip a number of rows | | skip-while condition | Skips rows while the condition matches. | +| split-by column | Creates a new table with the data from the inner tables splitted by the column given | | sort-by ...columns | Sort by the given columns | | str (column) | Apply string function. Optionally use the column of a table | | sum | Sum a column of values | diff --git a/src/cli.rs b/src/cli.rs index c6995ef711..b882d57d69 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -301,6 +301,7 @@ pub async fn cli() -> Result<(), Box> { whole_stream_command(FromYML), whole_stream_command(Pick), whole_stream_command(Get), + whole_stream_command(Histogram), per_item_command(Remove), per_item_command(Fetch), per_item_command(Open), @@ -320,6 +321,7 @@ pub async fn cli() -> Result<(), Box> { per_item_command(Mkdir), per_item_command(Move), whole_stream_command(Save), + whole_stream_command(SplitBy), whole_stream_command(Table), whole_stream_command(Version), whole_stream_command(Which), @@ -328,12 +330,10 @@ pub async fn cli() -> Result<(), Box> { cfg_if::cfg_if! { if #[cfg(data_processing_primitives)] { context.add_commands(vec![ - whole_stream_command(SplitBy), whole_stream_command(ReduceBy), whole_stream_command(EvaluateBy), whole_stream_command(TSortBy), whole_stream_command(MapMaxBy), - whole_stream_command(Histogram), ]); } } diff --git a/src/commands.rs b/src/commands.rs index 629289b565..ee70534640 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -16,6 +16,8 @@ pub(crate) mod debug; pub(crate) mod echo; pub(crate) mod enter; pub(crate) mod env; +#[allow(unused)] +pub(crate) mod evaluate_by; pub(crate) mod exit; pub(crate) mod fetch; pub(crate) mod first; @@ -33,10 +35,13 @@ pub(crate) mod from_yaml; pub(crate) mod get; pub(crate) mod group_by; pub(crate) mod help; +pub(crate) mod histogram; pub(crate) mod history; pub(crate) mod last; pub(crate) mod lines; pub(crate) mod ls; +#[allow(unused)] +pub(crate) mod map_max_by; pub(crate) mod mkdir; pub(crate) mod mv; pub(crate) mod next; @@ -49,6 +54,8 @@ pub(crate) mod post; pub(crate) mod prepend; pub(crate) mod prev; pub(crate) mod pwd; +#[allow(unused)] +pub(crate) mod reduce_by; pub(crate) mod reject; pub(crate) mod reverse; pub(crate) mod rm; @@ -57,20 +64,11 @@ pub(crate) mod shells; pub(crate) mod size; pub(crate) mod skip_while; pub(crate) mod sort_by; - -cfg_if::cfg_if! { - if #[cfg(data_processing_primitives)] { - pub(crate) mod split_by; - pub(crate) mod reduce_by; - pub(crate) mod evaluate_by; - pub(crate) mod t_sort_by; - pub(crate) mod map_max_by; - pub(crate) mod histogram; - } -} - +pub(crate) mod split_by; pub(crate) mod split_column; pub(crate) mod split_row; +#[allow(unused)] +pub(crate) mod t_sort_by; pub(crate) mod table; pub(crate) mod tags; pub(crate) mod to_bson; @@ -103,6 +101,8 @@ pub(crate) use debug::Debug; pub(crate) use echo::Echo; pub(crate) use enter::Enter; pub(crate) use env::Env; +#[allow(unused)] +pub(crate) use evaluate_by::EvaluateBy; pub(crate) use exit::Exit; pub(crate) use fetch::Fetch; pub(crate) use first::First; @@ -122,10 +122,13 @@ pub(crate) use from_yaml::FromYML; pub(crate) use get::Get; pub(crate) use group_by::GroupBy; pub(crate) use help::Help; +pub(crate) use histogram::Histogram; pub(crate) use history::History; pub(crate) use last::Last; pub(crate) use lines::Lines; pub(crate) use ls::LS; +#[allow(unused)] +pub(crate) use map_max_by::MapMaxBy; pub(crate) use mkdir::Mkdir; pub(crate) use mv::Move; pub(crate) use next::Next; @@ -137,6 +140,8 @@ pub(crate) use post::Post; pub(crate) use prepend::Prepend; pub(crate) use prev::Previous; pub(crate) use pwd::PWD; +#[allow(unused)] +pub(crate) use reduce_by::ReduceBy; pub(crate) use reject::Reject; pub(crate) use reverse::Reverse; pub(crate) use rm::Remove; @@ -145,20 +150,11 @@ pub(crate) use shells::Shells; pub(crate) use size::Size; pub(crate) use skip_while::SkipWhile; pub(crate) use sort_by::SortBy; - -cfg_if::cfg_if! { - if #[cfg(data_processing_primitives)] { - pub(crate) use split_by::SplitBy; - pub(crate) use reduce_by::ReduceBy; - pub(crate) use evaluate_by::EvaluateBy; - pub(crate) use t_sort_by::TSortBy; - pub(crate) use map_max_by::MapMaxBy; - pub(crate) use histogram::Histogram; - } -} - +pub(crate) use split_by::SplitBy; pub(crate) use split_column::SplitColumn; pub(crate) use split_row::SplitRow; +#[allow(unused)] +pub(crate) use t_sort_by::TSortBy; pub(crate) use table::Table; pub(crate) use tags::Tags; pub(crate) use to_bson::ToBSON; diff --git a/src/commands/histogram.rs b/src/commands/histogram.rs index 52d72bdfbf..6933f28a6f 100644 --- a/src/commands/histogram.rs +++ b/src/commands/histogram.rs @@ -1,10 +1,10 @@ -use crate::commands::WholeStreamCommand; +use crate::commands::evaluate_by::evaluate; use crate::commands::group_by::group; +use crate::commands::map_max_by::map_max; +use crate::commands::reduce_by::reduce; use crate::commands::t_sort_by::columns_sorted; use crate::commands::t_sort_by::t_sort; -use crate::commands::evaluate_by::evaluate; -use crate::commands::reduce_by::reduce; -use crate::commands::map_max_by::map_max; +use crate::commands::WholeStreamCommand; use crate::data::TaggedDictBuilder; use crate::errors::ShellError; use crate::prelude::*; @@ -15,6 +15,7 @@ pub struct Histogram; #[derive(Deserialize)] pub struct HistogramArgs { column_name: Tagged, + rest: Vec>, } impl WholeStreamCommand for Histogram { @@ -23,11 +24,16 @@ impl WholeStreamCommand for Histogram { } fn signature(&self) -> Signature { - Signature::build("histogram").required( - "column_name", - SyntaxShape::String, - "the name of the column to graph by", - ) + Signature::build("histogram") + .required( + "column_name", + SyntaxShape::String, + "the name of the column to graph by", + ) + .rest( + SyntaxShape::Member, + "column name to give the histogram's frequency column", + ) } fn usage(&self) -> &str { @@ -44,7 +50,7 @@ impl WholeStreamCommand for Histogram { } pub fn histogram( - HistogramArgs { column_name }: HistogramArgs, + HistogramArgs { column_name, rest }: HistogramArgs, RunnableContext { input, name, .. }: RunnableContext, ) -> Result { let stream = async_stream! { @@ -68,13 +74,24 @@ pub fn histogram( let mut idx = 0; + let column_names_supplied: Vec<_> = rest.iter().map(|f| f.item.clone()).collect(); + + let frequency_column_name = if column_names_supplied.is_empty() { + "frecuency".to_string() + } else { + column_names_supplied[0].clone() + }; + + let column = (*column_name).clone(); + if let Tagged { item: Value::Table(start), .. } = datasets.get(0).unwrap() { for percentage in start.into_iter() { + let mut fact = TaggedDictBuilder::new(&name); - fact.insert_tagged("committer", group_labels.get(idx).unwrap().clone()); + fact.insert_tagged(&column, group_labels.get(idx).unwrap().clone()); if let Tagged { item: Value::Primitive(Primitive::Int(ref num)), .. } = percentage.clone() { - fact.insert("activity", std::iter::repeat("*").take(num.to_i32().unwrap() as usize).collect::()); + fact.insert(&frequency_column_name, std::iter::repeat("*").take(num.to_i32().unwrap() as usize).collect::()); } idx = idx + 1; @@ -104,38 +121,36 @@ fn percentages( } => { let datasets: Vec<_> = datasets .into_iter() - .map(|subsets| { - match subsets { - Tagged { - item: Value::Table(data), - .. - } => { - let data = data - .into_iter() - .map(|d| match d { - Tagged { - item: Value::Primitive(Primitive::Int(n)), - .. - } => { - let max = match max { - Tagged { - item: Value::Primitive(Primitive::Int(ref maxima)), - .. - } => maxima.to_i32().unwrap(), - _ => 0, - }; + .map(|subsets| match subsets { + Tagged { + item: Value::Table(data), + .. + } => { + let data = data + .into_iter() + .map(|d| match d { + Tagged { + item: Value::Primitive(Primitive::Int(n)), + .. + } => { + let max = match max { + Tagged { + item: Value::Primitive(Primitive::Int(ref maxima)), + .. + } => maxima.to_i32().unwrap(), + _ => 0, + }; - let n = { n.to_i32().unwrap() * 100 / max }; + let n = { n.to_i32().unwrap() * 100 / max }; - Value::number(n).tagged(&tag) - } - _ => Value::number(0).tagged(&tag), - }) - .collect::>(); - Value::Table(data).tagged(&tag) - } - _ => Value::Table(vec![]).tagged(&tag), + Value::number(n).tagged(&tag) + } + _ => Value::number(0).tagged(&tag), + }) + .collect::>(); + Value::Table(data).tagged(&tag) } + _ => Value::Table(vec![]).tagged(&tag), }) .collect(); diff --git a/src/commands/map_max_by.rs b/src/commands/map_max_by.rs index 31a02a81b1..ea2fc99219 100644 --- a/src/commands/map_max_by.rs +++ b/src/commands/map_max_by.rs @@ -81,29 +81,27 @@ pub fn map_max( } => { let datasets: Vec<_> = datasets .into_iter() - .map(|subsets| { - match subsets { - Tagged { - item: Value::Table(data), - .. - } => { - let data = data.into_iter().fold(0, |acc, value| match value { - Tagged { - item: Value::Primitive(Primitive::Int(n)), - .. - } => { - if n.to_i32().unwrap() > acc { - n.to_i32().unwrap() - } else { - acc - } + .map(|subsets| match subsets { + Tagged { + item: Value::Table(data), + .. + } => { + let data = data.into_iter().fold(0, |acc, value| match value { + Tagged { + item: Value::Primitive(Primitive::Int(n)), + .. + } => { + if n.to_i32().unwrap() > acc { + n.to_i32().unwrap() + } else { + acc } - _ => acc, - }); - Value::number(data).tagged(&tag) - } - _ => Value::number(0).tagged(&tag), + } + _ => acc, + }); + Value::number(data).tagged(&tag) } + _ => Value::number(0).tagged(&tag), }) .collect(); diff --git a/src/commands/t_sort_by.rs b/src/commands/t_sort_by.rs index 1df4cce887..1c914dbac3 100644 --- a/src/commands/t_sort_by.rs +++ b/src/commands/t_sort_by.rs @@ -57,25 +57,25 @@ fn t_sort_by( RunnableContext { input, name, .. }: RunnableContext, ) -> Result { Ok(OutputStream::new(async_stream! { - let values: Vec> = input.values.collect().await; + let values: Vec> = input.values.collect().await; - let column_grouped_by_name = if let Some(grouped_by) = group_by { - Some(grouped_by.item().clone()) - } else { - None - }; + let column_grouped_by_name = if let Some(grouped_by) = group_by { + Some(grouped_by.item().clone()) + } else { + None + }; - if show_columns { - for label in columns_sorted(column_grouped_by_name, &values[0], &name).iter() { - yield ReturnSuccess::value(label.clone()); - } - } else { - match t_sort(column_grouped_by_name, None, &values[0], name) { - Ok(sorted) => yield ReturnSuccess::value(sorted), - Err(err) => yield Err(err) - } + if show_columns { + for label in columns_sorted(column_grouped_by_name, &values[0], &name).iter() { + yield ReturnSuccess::value(label.clone()); } - })) + } else { + match t_sort(column_grouped_by_name, None, &values[0], name) { + Ok(sorted) => yield ReturnSuccess::value(sorted), + Err(err) => yield Err(err) + } + } + })) } pub fn columns_sorted( @@ -125,7 +125,7 @@ pub fn columns_sorted( keys.into_iter().map(|k| k.tagged(&origin_tag)).collect() } - _ => vec![Value::string("default").tagged(&origin_tag)] + _ => vec![Value::string("default").tagged(&origin_tag)], } } @@ -238,7 +238,7 @@ mod tests { use crate::data::meta::*; use crate::Value; use indexmap::IndexMap; - + fn string(input: impl Into) -> Tagged { Value::string(input.into()).tagged_unknown() } @@ -305,7 +305,7 @@ mod tests { ] ) } - + #[test] fn sorts_the_tables() { let group_by = String::from("date"); diff --git a/tests/commands_test.rs b/tests/commands_test.rs index acd5e8374c..661d14023e 100644 --- a/tests/commands_test.rs +++ b/tests/commands_test.rs @@ -31,6 +31,35 @@ fn group_by() { }) } +#[test] +fn histogram() { + Playground::setup("histogram_test_1", |dirs, sandbox| { + sandbox.with_files(vec![FileWithContentToBeTrimmed( + "los_tres_caballeros.csv", + r#" + first_name,last_name,rusty_at + Andrés,Robalino,Ecuador + Jonathan,Turner,Estados Unidos + Yehuda,Katz,Estados Unidos + "#, + )]); + + let actual = nu!( + cwd: dirs.test(), h::pipeline( + r#" + open los_tres_caballeros.csv + | histogram rusty_at countries + | where rusty_at == "Ecuador" + | get countries + | echo $it + "# + )); + + assert_eq!(actual, "**************************************************"); + // 50% + }) +} + #[test] fn group_by_errors_if_unknown_column_name() { Playground::setup("group_by_test_2", |dirs, sandbox| { @@ -56,7 +85,6 @@ fn group_by_errors_if_unknown_column_name() { }) } -#[cfg(data_processing_primitives)] #[test] fn split_by() { Playground::setup("split_by_test_1", |dirs, sandbox| { @@ -86,7 +114,6 @@ fn split_by() { }) } -#[cfg(data_processing_primitives)] #[test] fn split_by_errors_if_no_table_given_as_input() { Playground::setup("split_by_test_2", |dirs, sandbox| {