Mirrors/nushell
2
0
Fork 0
mirror of https://github.com/nushell/nushell synced 2025-01-26 11:55:20 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #990 from drmason13/combine-csv-and-tsv

Browse source 
combine functions behind to/from-c/tsv commands
This commit is contained in:
Andrés N. Robalino 2019-11-19 11:29:33 -05:00 committed by GitHub
commit 388fc24191
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 365 additions and 394 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docs/commands/from-csv.md
View file

@ -92,6 +92,10 @@ error: Expected a string from pipeline
━━━┷━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━ ━━━┷━━━━━━━━━━━┷━━━━━━━━━┷━━━━━━
``` ```
The string '\t' can be used to separate on tabs. Note that this is the same as using the from-tsv command.
Newlines '\n' are not acceptable separators.
Note that separators are currently provided as strings and need to be wrapped in quotes. Note that separators are currently provided as strings and need to be wrapped in quotes.
```shell ```shell

124
docs/commands/to-csv.md
View file

@ -7,11 +7,11 @@ Converts table data into csv text.
```shell ```shell
> shells > shells
━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━ ━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━
# │ │ name │ path # │ │ name │ path
───┼───┼────────────┼──────────────────────── ───┼───┼────────────┼────────────────────────
0 │ X │ filesystem │ /home/shaurya 0 │ X │ filesystem │ /home/shaurya
1 │ │ filesystem │ /home/shaurya/Pictures 1 │ │ filesystem │ /home/shaurya/Pictures
2 │ │ filesystem │ /home/shaurya/Desktop 2 │ │ filesystem │ /home/shaurya/Desktop
━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━ ━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━
> shells | to-csv > shells | to-csv
,name,path ,name,path
@ -23,48 +23,48 @@ X,filesystem,/home/shaurya
```shell ```shell
> open caco3_plastics.csv > open caco3_plastics.csv
━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━━━━ ━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━━━━
# │ importer │ shipper │ tariff_item │ name │ origin │ shipped_at │ arrived_at │ net_weight │ fob_price │ cif_price │ cif_per_net_ # │ importer │ shipper │ tariff_item │ name │ origin │ shipped_at │ arrived_at │ net_weight │ fob_price │ cif_price │ cif_per_net_
│ │ │ │ │ │ │ │ │ │ │ weight │ │ │ │ │ │ │ │ │ │ │ weight
───┼──────────────┼──────────────┼─────────────┼──────────────┼──────────┼────────────┼────────────┼────────────┼───────────┼───────────┼────────────── ───┼──────────────┼──────────────┼─────────────┼──────────────┼──────────┼────────────┼────────────┼────────────┼───────────┼───────────┼──────────────
0 │ PLASTICOS │ S A REVERTE │ 2509000000 │ CARBONATO DE │ SPAIN │ 18/03/2016 │ 17/04/2016 │ 81,000.00 │ 14,417.58 │ 18,252.34 │ 0.23 0 │ PLASTICOS │ S A REVERTE │ 2509000000 │ CARBONATO DE │ SPAIN │ 18/03/2016 │ 17/04/2016 │ 81,000.00 │ 14,417.58 │ 18,252.34 │ 0.23
│ RIVAL CIA │ │ │ CALCIO TIPO │ │ │ │ │ │ │ │ RIVAL CIA │ │ │ CALCIO TIPO │ │ │ │ │ │ │
│ LTDA │ │ │ CALCIPORE │ │ │ │ │ │ │ │ LTDA │ │ │ CALCIPORE │ │ │ │ │ │ │
│ │ │ │ 160 T AL │ │ │ │ │ │ │ │ │ │ │ 160 T AL │ │ │ │ │ │ │
1 │ MEXICHEM │ OMYA ANDINA │ 2836500000 │ CARBONATO │ COLOMBIA │ 07/07/2016 │ 10/07/2016 │ 26,000.00 │ 7,072.00 │ 8,127.18 │ 0.31 1 │ MEXICHEM │ OMYA ANDINA │ 2836500000 │ CARBONATO │ COLOMBIA │ 07/07/2016 │ 10/07/2016 │ 26,000.00 │ 7,072.00 │ 8,127.18 │ 0.31
│ ECUADOR S.A. │ S A │ │ │ │ │ │ │ │ │ │ ECUADOR S.A. │ S A │ │ │ │ │ │ │ │ │
2 │ PLASTIAZUAY │ SA REVERTE │ 2836500000 │ CARBONATO DE │ SPAIN │ 27/07/2016 │ 09/08/2016 │ 81,000.00 │ 8,100.00 │ 11,474.55 │ 0.14 2 │ PLASTIAZUAY │ SA REVERTE │ 2836500000 │ CARBONATO DE │ SPAIN │ 27/07/2016 │ 09/08/2016 │ 81,000.00 │ 8,100.00 │ 11,474.55 │ 0.14
│ SA │ │ │ CALCIO │ │ │ │ │ │ │ │ SA │ │ │ CALCIO │ │ │ │ │ │ │
3 │ PLASTICOS │ AND │ 2836500000 │ CALCIUM │ TURKEY │ 04/10/2016 │ 11/11/2016 │ 100,000.00 │ 17,500.00 │ 22,533.75 │ 0.23 3 │ PLASTICOS │ AND │ 2836500000 │ CALCIUM │ TURKEY │ 04/10/2016 │ 11/11/2016 │ 100,000.00 │ 17,500.00 │ 22,533.75 │ 0.23
│ RIVAL CIA │ ENDUSTRIYEL │ │ CARBONATE │ │ │ │ │ │ │ │ RIVAL CIA │ ENDUSTRIYEL │ │ CARBONATE │ │ │ │ │ │ │
│ LTDA │ HAMMADDELER │ │ ANADOLU │ │ │ │ │ │ │ │ LTDA │ HAMMADDELER │ │ ANADOLU │ │ │ │ │ │ │
│ │ DIS TCARET │ │ ANDCARB CT-1 │ │ │ │ │ │ │ │ │ DIS TCARET │ │ ANDCARB CT-1 │ │ │ │ │ │ │
│ │ LTD.STI. │ │ │ │ │ │ │ │ │ │ │ LTD.STI. │ │ │ │ │ │ │ │ │
4 │ QUIMICA │ SA REVERTE │ 2836500000 │ CARBONATO DE │ SPAIN │ 24/06/2016 │ 12/07/2016 │ 27,000.00 │ 3,258.90 │ 5,585.00 │ 0.21 4 │ QUIMICA │ SA REVERTE │ 2836500000 │ CARBONATO DE │ SPAIN │ 24/06/2016 │ 12/07/2016 │ 27,000.00 │ 3,258.90 │ 5,585.00 │ 0.21
│ COMERCIAL │ │ │ CALCIO │ │ │ │ │ │ │ │ COMERCIAL │ │ │ CALCIO │ │ │ │ │ │ │
│ QUIMICIAL │ │ │ │ │ │ │ │ │ │ │ QUIMICIAL │ │ │ │ │ │ │ │ │ │
│ CIA. LTDA. │ │ │ │ │ │ │ │ │ │ │ CIA. LTDA. │ │ │ │ │ │ │ │ │ │
5 │ PICA │ OMYA ANDINA │ 3824909999 │ CARBONATO DE │ COLOMBIA │ 01/01/1900 │ 18/01/2016 │ 66,500.00 │ 12,635.00 │ 18,670.52 │ 0.28 5 │ PICA │ OMYA ANDINA │ 3824909999 │ CARBONATO DE │ COLOMBIA │ 01/01/1900 │ 18/01/2016 │ 66,500.00 │ 12,635.00 │ 18,670.52 │ 0.28
│ PLASTICOS │ S.A │ │ CALCIO │ │ │ │ │ │ │ │ PLASTICOS │ S.A │ │ CALCIO │ │ │ │ │ │ │
│ INDUSTRIALES │ │ │ │ │ │ │ │ │ │ │ INDUSTRIALES │ │ │ │ │ │ │ │ │ │
│ C.A. │ │ │ │ │ │ │ │ │ │ │ C.A. │ │ │ │ │ │ │ │ │ │
6 │ PLASTIQUIM │ OMYA ANDINA │ 3824909999 │ CARBONATO DE │ COLOMBIA │ 01/01/1900 │ 25/10/2016 │ 33,000.00 │ 6,270.00 │ 9,999.00 │ 0.30 6 │ PLASTIQUIM │ OMYA ANDINA │ 3824909999 │ CARBONATO DE │ COLOMBIA │ 01/01/1900 │ 25/10/2016 │ 33,000.00 │ 6,270.00 │ 9,999.00 │ 0.30
│ S.A. │ S.A NIT │ │ CALCIO │ │ │ │ │ │ │ │ S.A. │ S.A NIT │ │ CALCIO │ │ │ │ │ │ │
│ │ 830.027.386- │ │ RECUBIERTO │ │ │ │ │ │ │ │ │ 830.027.386- │ │ RECUBIERTO │ │ │ │ │ │ │
│ │ 6 │ │ CON ACIDO │ │ │ │ │ │ │ │ │ 6 │ │ CON ACIDO │ │ │ │ │ │ │
│ │ │ │ ESTEARICO │ │ │ │ │ │ │ │ │ │ │ ESTEARICO │ │ │ │ │ │ │
│ │ │ │ OMYA CARB 1T │ │ │ │ │ │ │ │ │ │ │ OMYA CARB 1T │ │ │ │ │ │ │
│ │ │ │ CG BBS 1000 │ │ │ │ │ │ │ │ │ │ │ CG BBS 1000 │ │ │ │ │ │ │
7 │ QUIMICOS │ SIBELCO │ 3824909999 │ CARBONATO DE │ COLOMBIA │ 01/11/2016 │ 03/11/2016 │ 52,000.00 │ 8,944.00 │ 13,039.05 │ 0.25 7 │ QUIMICOS │ SIBELCO │ 3824909999 │ CARBONATO DE │ COLOMBIA │ 01/11/2016 │ 03/11/2016 │ 52,000.00 │ 8,944.00 │ 13,039.05 │ 0.25
│ ANDINOS │ COLOMBIA SAS │ │ CALCIO │ │ │ │ │ │ │ │ ANDINOS │ COLOMBIA SAS │ │ CALCIO │ │ │ │ │ │ │
│ QUIMANDI │ │ │ RECUBIERTO │ │ │ │ │ │ │ │ QUIMANDI │ │ │ RECUBIERTO │ │ │ │ │ │ │
│ S.A. │ │ │ │ │ │ │ │ │ │ │ S.A. │ │ │ │ │ │ │ │ │ │
8 │ TIGRE │ OMYA ANDINA │ 3824909999 │ CARBONATO DE │ COLOMBIA │ 01/01/1900 │ 28/10/2016 │ 66,000.00 │ 11,748.00 │ 18,216.00 │ 0.28 8 │ TIGRE │ OMYA ANDINA │ 3824909999 │ CARBONATO DE │ COLOMBIA │ 01/01/1900 │ 28/10/2016 │ 66,000.00 │ 11,748.00 │ 18,216.00 │ 0.28
│ ECUADOR S.A. │ S.A NIT │ │ CALCIO │ │ │ │ │ │ │ │ ECUADOR S.A. │ S.A NIT │ │ CALCIO │ │ │ │ │ │ │
│ ECUATIGRE │ 830.027.386- │ │ RECUBIERTO │ │ │ │ │ │ │ │ ECUATIGRE │ 830.027.386- │ │ RECUBIERTO │ │ │ │ │ │ │
│ │ 6 │ │ CON ACIDO │ │ │ │ │ │ │ │ │ 6 │ │ CON ACIDO │ │ │ │ │ │ │
│ │ │ │ ESTEARICO │ │ │ │ │ │ │ │ │ │ │ ESTEARICO │ │ │ │ │ │ │
│ │ │ │ OMYACARB 1T │ │ │ │ │ │ │ │ │ │ │ OMYACARB 1T │ │ │ │ │ │ │
│ │ │ │ CG BPA 25 NO │ │ │ │ │ │ │ │ │ │ │ CG BPA 25 NO │ │ │ │ │ │ │
━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━━━━ ━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━━━━
> open caco3_plastics.csv | to-csv > open caco3_plastics.csv | to-csv
importer,shipper,tariff_item,name,origin,shipped_at,arrived_at,net_weight,fob_price,cif_price,cif_per_net_weight importer,shipper,tariff_item,name,origin,shipped_at,arrived_at,net_weight,fob_price,cif_price,cif_per_net_weight
@ -78,3 +78,37 @@ PLASTIQUIM S.A.,OMYA ANDINA S.A NIT 830.027.386-6,3824909999,CARBONATO DE CALCIO
QUIMICOS ANDINOS QUIMANDI S.A.,SIBELCO COLOMBIA SAS,3824909999,CARBONATO DE CALCIO RECUBIERTO,COLOMBIA,01/11/2016,03/11/2016,"52,000.00","8,944.00","13,039.05",0.25 QUIMICOS ANDINOS QUIMANDI S.A.,SIBELCO COLOMBIA SAS,3824909999,CARBONATO DE CALCIO RECUBIERTO,COLOMBIA,01/11/2016,03/11/2016,"52,000.00","8,944.00","13,039.05",0.25
TIGRE ECUADOR S.A. ECUATIGRE,OMYA ANDINA S.A NIT 830.027.386-6,3824909999,CARBONATO DE CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYACARB 1T CG BPA 25 NO,COLOMBIA,01/01/1900,28/10/2016,"66,000.00","11,748.00","18,216.00",0.28 TIGRE ECUADOR S.A. ECUATIGRE,OMYA ANDINA S.A NIT 830.027.386-6,3824909999,CARBONATO DE CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYACARB 1T CG BPA 25 NO,COLOMBIA,01/01/1900,28/10/2016,"66,000.00","11,748.00","18,216.00",0.28
``` ```
To use a character other than ',' to separate records, use `--separator` :
```shell
> shells
━━━┯━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━
# │ │ name │ path
───┼───┼────────────┼────────────────────────
0 │ X │ filesystem │ /home/shaurya
1 │ │ filesystem │ /home/shaurya/Pictures
2 │ │ filesystem │ /home/shaurya/Desktop
━━━┷━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━
> shells | to-csv --separator ';'
;name,path
X;filesystem;/home/shaurya
;filesystem;/home/shaurya/Pictures
;filesystem;/home/shaurya/Desktop
```
The string '\t' can be used to separate on tabs. Note that this is the same as using the to-tsv command.
Newlines '\n' are not acceptable separators.
Note that separators are currently provided as strings and need to be wrapped in quotes.
It is also considered an error to use a separator greater than one char :
```shell
> open pets.txt | from-csv --separator '123'
error: Expected a single separator char from --separator
- shell:1:37
1 | open pets.txt | from-csv --separator '123'
| ^^^^^ requires a single character string input
```

3
src/commands.rs
View file

@ -1,7 +1,8 @@
#[macro_use] #[macro_use]
pub(crate) mod macros; pub(crate) mod macros;
mod from_structured_data; mod from_delimited_data;
mod to_delimited_data;
pub(crate) mod append; pub(crate) mod append;
pub(crate) mod args; pub(crate) mod args;

26
src/commands/from_csv.rs
View file

@ -1,4 +1,4 @@
use crate::commands::from_structured_data::from_structured_data; use crate::commands::from_delimited_data::from_delimited_data;
use crate::commands::WholeStreamCommand; use crate::commands::WholeStreamCommand;
use crate::data::{Primitive, Value}; use crate::data::{Primitive, Value};
use crate::prelude::*; use crate::prelude::*;
@ -52,18 +52,22 @@ fn from_csv(
tag, tag,
.. ..
}) => { }) => {
let vec_s: Vec<char> = s.chars().collect(); if s == r"\t" {
if vec_s.len() != 1 { '\t'
return Err(ShellError::labeled_error( } else {
"Expected a single separator char from --separator", let vec_s: Vec<char> = s.chars().collect();
"requires a single character string input", if vec_s.len() != 1 {
tag, return Err(ShellError::labeled_error(
)); "Expected a single separator char from --separator",
}; "requires a single character string input",
vec_s[0] tag,
));
};
vec_s[0]
}
} }
_ => ',', _ => ',',
}; };
from_structured_data(headerless, sep, "CSV", runnable_context) from_delimited_data(headerless, sep, "CSV", runnable_context)
} }

6
src/commands/from_structured_data.rs → src/commands/from_delimited_data.rs
View file

@ -2,7 +2,7 @@ use crate::data::{Primitive, TaggedDictBuilder, Value};
use crate::prelude::*; use crate::prelude::*;
use csv::ReaderBuilder; use csv::ReaderBuilder;
fn from_stuctured_string_to_value( fn from_delimited_string_to_value(
s: String, s: String,
headerless: bool, headerless: bool,
separator: char, separator: char,
@ -37,7 +37,7 @@ fn from_stuctured_string_to_value(
Ok(Value::Table(rows).tagged(&tag)) Ok(Value::Table(rows).tagged(&tag))
} }
pub fn from_structured_data( pub fn from_delimited_data(
headerless: bool, headerless: bool,
sep: char, sep: char,
format_name: &'static str, format_name: &'static str,
@ -70,7 +70,7 @@ pub fn from_structured_data(
} }
} }
match from_stuctured_string_to_value(concat_string, headerless, sep, name_tag.clone()) { match from_delimited_string_to_value(concat_string, headerless, sep, name_tag.clone()) {
Ok(x) => match x { Ok(x) => match x {
Tagged { item: Value::Table(list), .. } => { Tagged { item: Value::Table(list), .. } => {
for l in list { for l in list {

2
src/commands/from_ssv.rs
View file

@ -488,7 +488,7 @@ mod tests {
#[test] #[test]
fn input_is_parsed_correctly_if_either_option_works() { fn input_is_parsed_correctly_if_either_option_works() {
let input = r#" let input = r#"
docker-registry docker-registry=default docker-registry=default 172.30.78.158 5000/TCP docker-registry docker-registry=default docker-registry=default 172.30.78.158 5000/TCP
kubernetes component=apiserver,provider=kubernetes <none> 172.30.0.2 443/TCP kubernetes component=apiserver,provider=kubernetes <none> 172.30.0.2 443/TCP
kubernetes-ro component=apiserver,provider=kubernetes <none> 172.30.0.1 80/TCP kubernetes-ro component=apiserver,provider=kubernetes <none> 172.30.0.1 80/TCP

4
src/commands/from_tsv.rs
View file

@ -1,4 +1,4 @@
use crate::commands::from_structured_data::from_structured_data; use crate::commands::from_delimited_data::from_delimited_data;
use crate::commands::WholeStreamCommand; use crate::commands::WholeStreamCommand;
use crate::prelude::*; use crate::prelude::*;
@ -36,5 +36,5 @@ fn from_tsv(
FromTSVArgs { headerless }: FromTSVArgs, FromTSVArgs { headerless }: FromTSVArgs,
runnable_context: RunnableContext, runnable_context: RunnableContext,
) -> Result<OutputStream, ShellError> { ) -> Result<OutputStream, ShellError> {
from_structured_data(headerless, '\t', "TSV", runnable_context) from_delimited_data(headerless, '\t', "TSV", runnable_context)
} }

193
src/commands/to_csv.rs
View file

@ -1,13 +1,14 @@
use crate::commands::to_delimited_data::to_delimited_data;
use crate::commands::WholeStreamCommand; use crate::commands::WholeStreamCommand;
use crate::data::{Primitive, Value}; use crate::data::{Primitive, Value};
use crate::prelude::*; use crate::prelude::*;
use csv::WriterBuilder;
pub struct ToCSV; pub struct ToCSV;
#[derive(Deserialize)] #[derive(Deserialize)]
pub struct ToCSVArgs { pub struct ToCSVArgs {
headerless: bool, headerless: bool,
separator: Option<Tagged<Value>>,
} }
impl WholeStreamCommand for ToCSV { impl WholeStreamCommand for ToCSV {
@ -35,170 +36,34 @@ impl WholeStreamCommand for ToCSV {
} }
} }
pub fn value_to_csv_value(v: &Tagged<Value>) -> Tagged<Value> {
match &v.item {
Value::Primitive(Primitive::String(s)) => Value::Primitive(Primitive::String(s.clone())),
Value::Primitive(Primitive::Nothing) => Value::Primitive(Primitive::Nothing),
Value::Primitive(Primitive::Boolean(b)) => Value::Primitive(Primitive::Boolean(b.clone())),
Value::Primitive(Primitive::Decimal(f)) => Value::Primitive(Primitive::Decimal(f.clone())),
Value::Primitive(Primitive::Int(i)) => Value::Primitive(Primitive::Int(i.clone())),
Value::Primitive(Primitive::Path(x)) => Value::Primitive(Primitive::Path(x.clone())),
Value::Primitive(Primitive::Bytes(b)) => Value::Primitive(Primitive::Bytes(b.clone())),
Value::Primitive(Primitive::Date(d)) => Value::Primitive(Primitive::Date(d.clone())),
Value::Row(o) => Value::Row(o.clone()),
Value::Table(l) => Value::Table(l.clone()),
Value::Block(_) => Value::Primitive(Primitive::Nothing),
_ => Value::Primitive(Primitive::Nothing),
}
.tagged(v.tag.clone())
}
fn to_string_helper(v: &Tagged<Value>) -> Result<String, ShellError> {
match &v.item {
Value::Primitive(Primitive::Date(d)) => Ok(d.to_string()),
Value::Primitive(Primitive::Bytes(b)) => Ok(format!("{}", b)),
Value::Primitive(Primitive::Boolean(_)) => Ok(v.as_string()?),
Value::Primitive(Primitive::Decimal(_)) => Ok(v.as_string()?),
Value::Primitive(Primitive::Int(_)) => Ok(v.as_string()?),
Value::Primitive(Primitive::Path(_)) => Ok(v.as_string()?),
Value::Table(_) => return Ok(String::from("[Table]")),
Value::Row(_) => return Ok(String::from("[Row]")),
Value::Primitive(Primitive::String(s)) => return Ok(s.to_string()),
_ => {
return Err(ShellError::labeled_error(
"Unexpected value",
"",
v.tag.clone(),
))
}
}
}
fn merge_descriptors(values: &[Tagged<Value>]) -> Vec<String> {
let mut ret = vec![];
for value in values {
for desc in value.data_descriptors() {
if !ret.contains(&desc) {
ret.push(desc);
}
}
}
ret
}
pub fn to_string(tagged_value: &Tagged<Value>) -> Result<String, ShellError> {
let v = &tagged_value.item;
match v {
Value::Row(o) => {
let mut wtr = WriterBuilder::new().from_writer(vec![]);
let mut fields: VecDeque<String> = VecDeque::new();
let mut values: VecDeque<String> = VecDeque::new();
for (k, v) in o.entries.iter() {
fields.push_back(k.clone());
values.push_back(to_string_helper(&v)?);
}
wtr.write_record(fields).expect("can not write.");
wtr.write_record(values).expect("can not write.");
return Ok(String::from_utf8(wtr.into_inner().map_err(|_| {
ShellError::labeled_error(
"Could not convert record",
"original value",
&tagged_value.tag,
)
})?)
.map_err(|_| {
ShellError::labeled_error(
"Could not convert record",
"original value",
&tagged_value.tag,
)
})?);
}
Value::Table(list) => {
let mut wtr = WriterBuilder::new().from_writer(vec![]);
let merged_descriptors = merge_descriptors(&list);
wtr.write_record(&merged_descriptors)
.expect("can not write.");
for l in list {
let mut row = vec![];
for desc in &merged_descriptors {
match l.item.get_data_by_key(&desc) {
Some(s) => {
row.push(to_string_helper(s)?);
}
None => {
row.push(String::new());
}
}
}
wtr.write_record(&row).expect("can not write");
}
return Ok(String::from_utf8(wtr.into_inner().map_err(|_| {
ShellError::labeled_error(
"Could not convert record",
"original value",
&tagged_value.tag,
)
})?)
.map_err(|_| {
ShellError::labeled_error(
"Could not convert record",
"original value",
&tagged_value.tag,
)
})?);
}
_ => return to_string_helper(tagged_value),
}
}
fn to_csv( fn to_csv(
ToCSVArgs { headerless }: ToCSVArgs, ToCSVArgs {
RunnableContext { input, name, .. }: RunnableContext, separator,
headerless,
}: ToCSVArgs,
runnable_context: RunnableContext,
) -> Result<OutputStream, ShellError> { ) -> Result<OutputStream, ShellError> {
let name_tag = name; let sep = match separator {
let stream = async_stream! { Some(Tagged {
let input: Vec<Tagged<Value>> = input.values.collect().await; item: Value::Primitive(Primitive::String(s)),
tag,
let to_process_input = if input.len() > 1 { ..
let tag = input[0].tag.clone(); }) => {
vec![Tagged { item: Value::Table(input), tag } ] if s == r"\t" {
} else if input.len() == 1 { '\t'
input } else {
} else { let vec_s: Vec<char> = s.chars().collect();
vec![] if vec_s.len() != 1 {
}; return Err(ShellError::labeled_error(
"Expected a single separator char from --separator",
for value in to_process_input { "requires a single character string input",
match to_string(&value_to_csv_value(&value)) { tag,
Ok(x) => { ));
let converted = if headerless { };
x.lines().skip(1).collect() vec_s[0]
} else { }
x }
}; _ => ',',
yield ReturnSuccess::value(Value::Primitive(Primitive::String(converted)).tagged(&name_tag))
}
_ => {
yield Err(ShellError::labeled_error_with_secondary(
"Expected a table with CSV-compatible structure.tag() from pipeline",
"requires CSV-compatible input",
&name_tag,
"originates from here".to_string(),
value.tag(),
))
}
}
}
}; };
to_delimited_data(headerless, sep, "CSV", runnable_context)
Ok(stream.to_output_stream())
} }

188
src/commands/to_delimited_data.rs Normal file
View file

@ -0,0 +1,188 @@
use crate::data::{Primitive, Value};
use crate::prelude::*;
use csv::WriterBuilder;
fn from_value_to_delimited_string(
tagged_value: &Tagged<Value>,
separator: char,
) -> Result<String, ShellError> {
let v = &tagged_value.item;
match v {
Value::Row(o) => {
let mut wtr = WriterBuilder::new()
.delimiter(separator as u8)
.from_writer(vec![]);
let mut fields: VecDeque<String> = VecDeque::new();
let mut values: VecDeque<String> = VecDeque::new();
for (k, v) in o.entries.iter() {
fields.push_back(k.clone());
values.push_back(to_string_tagged_value(&v)?);
}
wtr.write_record(fields).expect("can not write.");
wtr.write_record(values).expect("can not write.");
return Ok(String::from_utf8(wtr.into_inner().map_err(|_| {
ShellError::labeled_error(
"Could not convert record",
"original value",
&tagged_value.tag,
)
})?)
.map_err(|_| {
ShellError::labeled_error(
"Could not convert record",
"original value",
&tagged_value.tag,
)
})?);
}
Value::Table(list) => {
let mut wtr = WriterBuilder::new()
.delimiter(separator as u8)
.from_writer(vec![]);
let merged_descriptors = merge_descriptors(&list);
wtr.write_record(&merged_descriptors)
.expect("can not write.");
for l in list {
let mut row = vec![];
for desc in &merged_descriptors {
match l.item.get_data_by_key(&desc) {
Some(s) => {
row.push(to_string_tagged_value(s)?);
}
None => {
row.push(String::new());
}
}
}
wtr.write_record(&row).expect("can not write");
}
return Ok(String::from_utf8(wtr.into_inner().map_err(|_| {
ShellError::labeled_error(
"Could not convert record",
"original value",
&tagged_value.tag,
)
})?)
.map_err(|_| {
ShellError::labeled_error(
"Could not convert record",
"original value",
&tagged_value.tag,
)
})?);
}
_ => return to_string_tagged_value(tagged_value),
}
}
// NOTE: could this be useful more widely and implemented on Tagged<Value> ?
pub fn clone_tagged_value(v: &Tagged<Value>) -> Tagged<Value> {
match &v.item {
Value::Primitive(Primitive::String(s)) => Value::Primitive(Primitive::String(s.clone())),
Value::Primitive(Primitive::Nothing) => Value::Primitive(Primitive::Nothing),
Value::Primitive(Primitive::Boolean(b)) => Value::Primitive(Primitive::Boolean(b.clone())),
Value::Primitive(Primitive::Decimal(f)) => Value::Primitive(Primitive::Decimal(f.clone())),
Value::Primitive(Primitive::Int(i)) => Value::Primitive(Primitive::Int(i.clone())),
Value::Primitive(Primitive::Path(x)) => Value::Primitive(Primitive::Path(x.clone())),
Value::Primitive(Primitive::Bytes(b)) => Value::Primitive(Primitive::Bytes(b.clone())),
Value::Primitive(Primitive::Date(d)) => Value::Primitive(Primitive::Date(d.clone())),
Value::Row(o) => Value::Row(o.clone()),
Value::Table(l) => Value::Table(l.clone()),
Value::Block(_) => Value::Primitive(Primitive::Nothing),
_ => Value::Primitive(Primitive::Nothing),
}
.tagged(v.tag.clone())
}
// NOTE: could this be useful more widely and implemented on Tagged<Value> ?
fn to_string_tagged_value(v: &Tagged<Value>) -> Result<String, ShellError> {
match &v.item {
Value::Primitive(Primitive::Date(d)) => Ok(d.to_string()),
Value::Primitive(Primitive::Bytes(b)) => {
let tmp = format!("{}", b);
Ok(tmp)
}
Value::Primitive(Primitive::Boolean(_)) => Ok(v.as_string()?),
Value::Primitive(Primitive::Decimal(_)) => Ok(v.as_string()?),
Value::Primitive(Primitive::Int(_)) => Ok(v.as_string()?),
Value::Primitive(Primitive::Path(_)) => Ok(v.as_string()?),
Value::Table(_) => return Ok(String::from("[Table]")),
Value::Row(_) => return Ok(String::from("[Row]")),
Value::Primitive(Primitive::String(s)) => return Ok(s.to_string()),
_ => {
return Err(ShellError::labeled_error(
"Unexpected value",
"",
v.tag.clone(),
))
}
}
}
fn merge_descriptors(values: &[Tagged<Value>]) -> Vec<String> {
let mut ret = vec![];
for value in values {
for desc in value.data_descriptors() {
if !ret.contains(&desc) {
ret.push(desc);
}
}
}
ret
}
pub fn to_delimited_data(
headerless: bool,
sep: char,
format_name: &'static str,
RunnableContext { input, name, .. }: RunnableContext,
) -> Result<OutputStream, ShellError> {
let name_tag = name;
let stream = async_stream! {
let input: Vec<Tagged<Value>> = input.values.collect().await;
let to_process_input = if input.len() > 1 {
let tag = input[0].tag.clone();
vec![Tagged { item: Value::Table(input), tag } ]
} else if input.len() == 1 {
input
} else {
vec![]
};
for value in to_process_input {
match from_value_to_delimited_string(&clone_tagged_value(&value), sep) {
Ok(x) => {
let converted = if headerless {
x.lines().skip(1).collect()
} else {
x
};
yield ReturnSuccess::value(Value::Primitive(Primitive::String(converted)).tagged(&name_tag))
}
_ => {
let expected = format!("Expected a table with {}-compatible structure.tag() from pipeline", format_name);
let requires = format!("requires {}-compatible input", format_name);
yield Err(ShellError::labeled_error_with_secondary(
expected,
requires,
&name_tag,
"originates from here".to_string(),
value.tag(),
))
}
}
}
};
Ok(stream.to_output_stream())
}

170
src/commands/to_tsv.rs
View file

@ -1,7 +1,6 @@
use crate::commands::to_delimited_data::to_delimited_data;
use crate::commands::WholeStreamCommand; use crate::commands::WholeStreamCommand;
use crate::data::{Primitive, Value};
use crate::prelude::*; use crate::prelude::*;
use csv::WriterBuilder;
pub struct ToTSV; pub struct ToTSV;
@ -35,172 +34,9 @@ impl WholeStreamCommand for ToTSV {
} }
} }
pub fn value_to_tsv_value(tagged_value: &Tagged<Value>) -> Tagged<Value> {
let v = &tagged_value.item;
match v {
Value::Primitive(Primitive::String(s)) => Value::Primitive(Primitive::String(s.clone())),
Value::Primitive(Primitive::Nothing) => Value::Primitive(Primitive::Nothing),
Value::Primitive(Primitive::Boolean(b)) => Value::Primitive(Primitive::Boolean(b.clone())),
Value::Primitive(Primitive::Decimal(f)) => Value::Primitive(Primitive::Decimal(f.clone())),
Value::Primitive(Primitive::Int(i)) => Value::Primitive(Primitive::Int(i.clone())),
Value::Primitive(Primitive::Path(x)) => Value::Primitive(Primitive::Path(x.clone())),
Value::Primitive(Primitive::Bytes(b)) => Value::Primitive(Primitive::Bytes(b.clone())),
Value::Primitive(Primitive::Date(d)) => Value::Primitive(Primitive::Date(d.clone())),
Value::Row(o) => Value::Row(o.clone()),
Value::Table(l) => Value::Table(l.clone()),
Value::Block(_) => Value::Primitive(Primitive::Nothing),
_ => Value::Primitive(Primitive::Nothing),
}
.tagged(&tagged_value.tag)
}
fn to_string_helper(tagged_value: &Tagged<Value>) -> Result<String, ShellError> {
let v = &tagged_value.item;
match v {
Value::Primitive(Primitive::Date(d)) => Ok(d.to_string()),
Value::Primitive(Primitive::Bytes(b)) => Ok(format!("{}", b)),
Value::Primitive(Primitive::Boolean(_)) => Ok(tagged_value.as_string()?),
Value::Primitive(Primitive::Decimal(_)) => Ok(tagged_value.as_string()?),
Value::Primitive(Primitive::Int(_)) => Ok(tagged_value.as_string()?),
Value::Primitive(Primitive::Path(_)) => Ok(tagged_value.as_string()?),
Value::Table(_) => return Ok(String::from("[table]")),
Value::Row(_) => return Ok(String::from("[row]")),
Value::Primitive(Primitive::String(s)) => return Ok(s.to_string()),
_ => {
return Err(ShellError::labeled_error(
"Unexpected value",
"original value",
&tagged_value.tag,
))
}
}
}
fn merge_descriptors(values: &[Tagged<Value>]) -> Vec<String> {
let mut ret = vec![];
for value in values {
for desc in value.data_descriptors() {
if !ret.contains(&desc) {
ret.push(desc);
}
}
}
ret
}
pub fn to_string(tagged_value: &Tagged<Value>) -> Result<String, ShellError> {
let v = &tagged_value.item;
match v {
Value::Row(o) => {
let mut wtr = WriterBuilder::new().delimiter(b'\t').from_writer(vec![]);
let mut fields: VecDeque<String> = VecDeque::new();
let mut values: VecDeque<String> = VecDeque::new();
for (k, v) in o.entries.iter() {
fields.push_back(k.clone());
values.push_back(to_string_helper(&v)?);
}
wtr.write_record(fields).expect("can not write.");
wtr.write_record(values).expect("can not write.");
return Ok(String::from_utf8(wtr.into_inner().map_err(|_| {
ShellError::labeled_error(
"Could not convert record",
"original value",
&tagged_value.tag,
)
})?)
.map_err(|_| {
ShellError::labeled_error(
"Could not convert record",
"original value",
&tagged_value.tag,
)
})?);
}
Value::Table(list) => {
let mut wtr = WriterBuilder::new().delimiter(b'\t').from_writer(vec![]);
let merged_descriptors = merge_descriptors(&list);
wtr.write_record(&merged_descriptors)
.expect("can not write.");
for l in list {
let mut row = vec![];
for desc in &merged_descriptors {
match l.item.get_data_by_key(&desc) {
Some(s) => {
row.push(to_string_helper(s)?);
}
None => {
row.push(String::new());
}
}
}
wtr.write_record(&row).expect("can not write");
}
return Ok(String::from_utf8(wtr.into_inner().map_err(|_| {
ShellError::labeled_error(
"Could not convert record",
"original value",
&tagged_value.tag,
)
})?)
.map_err(|_| {
ShellError::labeled_error(
"Could not convert record",
"original value",
&tagged_value.tag,
)
})?);
}
_ => return to_string_helper(tagged_value),
}
}
fn to_tsv( fn to_tsv(
ToTSVArgs { headerless }: ToTSVArgs, ToTSVArgs { headerless }: ToTSVArgs,
RunnableContext { input, name, .. }: RunnableContext, runnable_context: RunnableContext,
) -> Result<OutputStream, ShellError> { ) -> Result<OutputStream, ShellError> {
let name_tag = name; to_delimited_data(headerless, '\t', "TSV", runnable_context)
let stream = async_stream! {
let input: Vec<Tagged<Value>> = input.values.collect().await;
let to_process_input = if input.len() > 1 {
let tag = input[0].tag.clone();
vec![Tagged { item: Value::Table(input), tag } ]
} else if input.len() == 1 {
input
} else {
vec![]
};
for value in to_process_input {
match to_string(&value_to_tsv_value(&value)) {
Ok(x) => {
let converted = if headerless {
x.lines().skip(1).collect()
} else {
x
};
yield ReturnSuccess::value(Value::Primitive(Primitive::String(converted)).tagged(&name_tag))
}
_ => {
yield Err(ShellError::labeled_error_with_secondary(
"Expected a table with TSV-compatible structure.tag() from pipeline",
"requires TSV-compatible input",
&name_tag,
"originates from here".to_string(),
value.tag(),
))
}
}
}
};
Ok(stream.to_output_stream())
} }

39
tests/filters_test.rs
View file

@ -129,6 +129,35 @@ fn converts_from_csv_text_with_separator_to_structured_table() {
}) })
} }
#[test]
fn converts_from_csv_text_with_tab_separator_to_structured_table() {
Playground::setup("filter_from_csv_test_1", |dirs, sandbox| {
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_caballeros.txt",
r#"
first_name last_name rusty_luck
Andrés Robalino 1
Jonathan Turner 1
Yehuda Katz 1
"#,
)]);
let actual = nu!(
cwd: dirs.test(), h::pipeline(
r#"
open los_tres_caballeros.txt
| from-csv --separator '\t'
| get rusty_luck
| str --to-int
| sum
| echo $it
"#
));
assert_eq!(actual, "3");
})
}
#[test] #[test]
fn converts_from_csv_text_skipping_headers_to_structured_table() { fn converts_from_csv_text_skipping_headers_to_structured_table() {
Playground::setup("filter_from_csv_test_2", |dirs, sandbox| { Playground::setup("filter_from_csv_test_2", |dirs, sandbox| {
@ -267,6 +296,16 @@ fn can_convert_table_to_tsv_text_and_from_tsv_text_back_into_table() {
assert_eq!(actual, "SPAIN"); assert_eq!(actual, "SPAIN");
} }
#[test]
fn can_convert_table_to_tsv_text_and_from_tsv_text_back_into_table_using_csv_separator() {
let actual = nu!(
cwd: "tests/fixtures/formats",
r"open caco3_plastics.tsv | to-tsv | from-csv --separator '\t' | first 1 | get origin | echo $it"
);
assert_eq!(actual, "SPAIN");
}
#[test] #[test]
fn converts_structured_table_to_tsv_text() { fn converts_structured_table_to_tsv_text() {
Playground::setup("filter_to_tsv_test_1", |dirs, sandbox| { Playground::setup("filter_to_tsv_test_1", |dirs, sandbox| {