Mirrors/nushell
2
0
Fork 0
mirror of https://github.com/nushell/nushell synced 2024-11-10 07:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity

from yaml and from yml (#322)

Browse source 
* MathEval Variance and Stddev

* Fix tests and linting

* Typo

* Deal with streams when they are not tables

* `from yaml` and `from yml`

`from yaml` and `from yml`

from yaml and from yml

* Fix collect_string

* Fix tests and linting
This commit is contained in:
Luccas Mateus 2021-11-09 22:02:33 -03:00 committed by GitHub
parent d094f654c3
commit 75cfee28b2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 329 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
Cargo.lock generated
View file

@ -420,6 +420,12 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
[[package]]
name = "dtoa"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
[[package]]
name = "dunce"
version = "1.0.2"
@ -491,6 +497,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]]
name = "hashbrown"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
[[package]]
name = "hermit-abi"
version = "0.1.19"
@ -514,6 +526,16 @@ dependencies = [
"version_check",
]
[[package]]
name = "indexmap"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5"
dependencies = [
"autocfg",
"hashbrown",
]
[[package]]
name = "instant"
version = "0.1.12"
@ -745,6 +767,7 @@ dependencies = [
"csv",
"dialoguer",
"glob",
"itertools",
"lscolors",
"meval",
"nu-engine",
@ -756,6 +779,8 @@ dependencies = [
"nu-term-grid",
"rand",
"rayon",
"serde",
"serde_yaml",
"sysinfo",
"terminal_size",
"thiserror",
@ -1285,6 +1310,18 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_yaml"
version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8c608a35705a5d3cdc9fbe403147647ff34b921f8e833e49306df898f9b20af"
dependencies = [
"dtoa",
"indexmap",
"serde",
"yaml-rust",
]
[[package]]
name = "signal-hook"
version = "0.3.10"
@ -1631,6 +1668,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "yaml-rust"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
dependencies = [
"linked-hash-map",
]
[[package]]
name = "zeroize"
version = "1.4.3"

3
crates/nu-command/Cargo.toml
View file

@ -33,6 +33,9 @@ dialoguer = "0.9.0"
rayon = "1.5.1"
titlecase = "1.1.0"
meval = "0.2.0"
serde = { version="1.0.123", features=["derive"] }
serde_yaml = "0.8.16"
itertools = "0.10.0"
rand = "0.8"
[features]

2
crates/nu-command/src/default_context.rs
View file

@ -47,6 +47,8 @@ pub fn create_default_context() -> EngineState {
From,
FromCsv,
FromJson,
FromYaml,
FromYml,
FromTsv,
Get,
Griddle,

3
crates/nu-command/src/formats/from/mod.rs
View file

@ -3,8 +3,11 @@ mod csv;
mod delimited;
mod json;
mod tsv;
mod yaml;
pub use self::csv::FromCsv;
pub use command::From;
pub use json::FromJson;
pub use tsv::FromTsv;
pub use yaml::FromYaml;
pub use yaml::FromYml;

275
crates/nu-command/src/formats/from/yaml.rs Normal file
View file

@ -0,0 +1,275 @@
use itertools::Itertools;
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{
Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Spanned, Value,
};
use serde::de::Deserialize;
use std::collections::HashMap;
#[derive(Clone)]
pub struct FromYaml;
impl Command for FromYaml {
fn name(&self) -> &str {
"from yaml"
}
fn signature(&self) -> Signature {
Signature::build("from yaml")
}
fn usage(&self) -> &str {
"Parse text as .yaml/.yml and create table."
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
example: "'a: 1' | from yaml",
description: "Converts yaml formatted string to table",
result: Some(Value::Record {
cols: vec!["a".to_string()],
vals: vec![Value::Int {
val: 1,
span: Span::unknown(),
}],
span: Span::unknown(),
}),
},
Example {
example: "'[ a: 1, b: [1, 2] ]' | from yaml",
description: "Converts yaml formatted string to table",
result: Some(Value::List {
vals: vec![
Value::Record {
cols: vec!["a".to_string()],
vals: vec![Value::test_int(1)],
span: Span::unknown(),
},
Value::Record {
cols: vec!["b".to_string()],
vals: vec![Value::List {
vals: vec![Value::test_int(1), Value::test_int(2)],
span: Span::unknown(),
}],
span: Span::unknown(),
},
],
span: Span::unknown(),
}),
},
]
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<nu_protocol::PipelineData, ShellError> {
let head = call.head;
from_yaml(input, head)
}
}
#[derive(Clone)]
pub struct FromYml;
impl Command for FromYml {
fn name(&self) -> &str {
"from yml"
}
fn signature(&self) -> Signature {
Signature::build("from yml")
}
fn usage(&self) -> &str {
"Parse text as .yaml/.yml and create table."
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<nu_protocol::PipelineData, ShellError> {
let head = call.head;
from_yaml(input, head)
}
}
fn convert_yaml_value_to_nu_value(v: &serde_yaml::Value, span: Span) -> Result<Value, ShellError> {
let err_not_compatible_number =
ShellError::UnsupportedInput("Expected a compatible number".to_string(), span);
Ok(match v {
serde_yaml::Value::Bool(b) => Value::Bool { val: *b, span },
serde_yaml::Value::Number(n) if n.is_i64() => Value::Int {
val: n.as_i64().ok_or(err_not_compatible_number)?,
span,
},
serde_yaml::Value::Number(n) if n.is_f64() => Value::Float {
val: n.as_f64().ok_or(err_not_compatible_number)?,
span,
},
serde_yaml::Value::String(s) => Value::String {
val: s.to_string(),
span,
},
serde_yaml::Value::Sequence(a) => {
let result: Result<Vec<Value>, ShellError> = a
.iter()
.map(|x| convert_yaml_value_to_nu_value(x, span))
.collect();
Value::List {
vals: result?,
span,
}
}
serde_yaml::Value::Mapping(t) => {
let mut collected = Spanned {
item: HashMap::new(),
span,
};
for (k, v) in t {
// A ShellError that we re-use multiple times in the Mapping scenario
let err_unexpected_map = ShellError::UnsupportedInput(
format!("Unexpected YAML:\nKey: {:?}\nValue: {:?}", k, v),
span,
);
match (k, v) {
(serde_yaml::Value::String(k), _) => {
collected
.item
.insert(k.clone(), convert_yaml_value_to_nu_value(v, span)?);
}
// Hard-code fix for cases where "v" is a string without quotations with double curly braces
// e.g. k = value
// value: {{ something }}
// Strangely, serde_yaml returns
// "value" -> Mapping(Mapping { map: {Mapping(Mapping { map: {String("something"): Null} }): Null} })
(serde_yaml::Value::Mapping(m), serde_yaml::Value::Null) => {
return m
.iter()
.take(1)
.collect_vec()
.first()
.and_then(|e| match e {
(serde_yaml::Value::String(s), serde_yaml::Value::Null) => {
Some(Value::String {
val: "{{ ".to_owned() + s + " }}",
span,
})
}
_ => None,
})
.ok_or(err_unexpected_map);
}
(_, _) => {
return Err(err_unexpected_map);
}
}
}
Value::from(collected)
}
serde_yaml::Value::Null => Value::nothing(span),
x => unimplemented!("Unsupported yaml case: {:?}", x),
})
}
pub fn from_yaml_string_to_value(s: String, span: Span) -> Result<Value, ShellError> {
let mut documents = vec![];
for document in serde_yaml::Deserializer::from_str(&s) {
let v: serde_yaml::Value = serde_yaml::Value::deserialize(document).map_err(|x| {
ShellError::UnsupportedInput(format!("Could not load yaml: {}", x), span)
})?;
documents.push(convert_yaml_value_to_nu_value(&v, span)?);
}
match documents.len() {
0 => Ok(Value::nothing(span)),
1 => Ok(documents.remove(0)),
_ => Ok(Value::List {
vals: documents,
span,
}),
}
}
fn from_yaml(input: PipelineData, head: Span) -> Result<PipelineData, ShellError> {
let concat_string = input.collect_string("");
match from_yaml_string_to_value(concat_string, head) {
Ok(x) => Ok(x.into_pipeline_data()),
Err(other) => Err(other),
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_problematic_yaml() {
struct TestCase {
description: &'static str,
input: &'static str,
expected: Result<Value, ShellError>,
}
let tt: Vec<TestCase> = vec![
TestCase {
description: "Double Curly Braces With Quotes",
input: r#"value: "{{ something }}""#,
expected: Ok(Value::Record {
cols: vec!["value".to_string()],
vals: vec![Value::String {
val: "{{ something }}".to_string(),
span: Span::unknown(),
}],
span: Span::unknown(),
}),
},
TestCase {
description: "Double Curly Braces Without Quotes",
input: r#"value: {{ something }}"#,
expected: Ok(Value::Record {
cols: vec!["value".to_string()],
vals: vec![Value::String {
val: "{{ something }}".to_string(),
span: Span::unknown(),
}],
span: Span::unknown(),
}),
},
];
for tc in tt {
let actual = from_yaml_string_to_value(tc.input.to_owned(), Span::unknown());
if actual.is_err() {
assert!(
tc.expected.is_err(),
"actual is Err for test:\nTest Description {}\nErr: {:?}",
tc.description,
actual
);
} else {
assert_eq!(
actual.unwrap().into_string(""),
tc.expected.unwrap().into_string("")
);
}
}
}
#[test]
fn test_examples() {
use crate::test_examples;
test_examples(FromYaml {})
}
}