Mirrors/nushell
2
0
Fork 0
mirror of https://github.com/nushell/nushell synced 2024-12-28 14:03:09 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add string stream and binary stream, add text decoding (#570)

Browse source 
* WIP

* Add binary/string streams and text decoding

* Make string collection fallible

* Oops, forgot pretty hex

* Oops, forgot pretty hex

* clippy
This commit is contained in:
JT 2021-12-24 18:22:11 +11:00 committed by GitHub
parent 7f0921a14b
commit 3522bead97
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
50 changed files with 1633 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
Cargo.lock generated
View file

@ -78,7 +78,7 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcb2392079bf27198570d6af79ecbd9ec7d8f16d3ec6b60933922fdb66287127" checksum = "bcb2392079bf27198570d6af79ecbd9ec7d8f16d3ec6b60933922fdb66287127"
dependencies = [ dependencies = [
"heapless", "heapless 0.5.6",
"nom 4.2.3", "nom 4.2.3",
] ]
@ -851,6 +851,7 @@ dependencies = [
"nu-parser", "nu-parser",
"nu-path", "nu-path",
"nu-plugin", "nu-plugin",
"nu-pretty-hex",
"nu-protocol", "nu-protocol",
"nu-table", "nu-table",
"nu-term-grid", "nu-term-grid",
@ -1102,6 +1103,15 @@ dependencies = [
"byteorder", "byteorder",
] ]
[[package]]
name = "hash32"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67"
dependencies = [
"byteorder",
]
[[package]] [[package]]
name = "hash_hasher" name = "hash_hasher"
version = "2.0.3" version = "2.0.3"
@ -1126,7 +1136,18 @@ checksum = "74911a68a1658cfcfb61bc0ccfbd536e3b6e906f8c2f7883ee50157e3e2184f1"
dependencies = [ dependencies = [
"as-slice", "as-slice",
"generic-array 0.13.3", "generic-array 0.13.3",
"hash32", "hash32 0.1.1",
"stable_deref_trait",
]
[[package]]
name = "heapless"
version = "0.7.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e476c64197665c3725621f0ac3f9e5209aa5e889e02a08b1daf5f16dc5fd952"
dependencies = [
"hash32 0.2.1",
"spin",
"stable_deref_trait", "stable_deref_trait",
] ]
@ -1690,6 +1711,7 @@ dependencies = [
"digest 0.10.0", "digest 0.10.0",
"dtparse", "dtparse",
"eml-parser", "eml-parser",
"encoding_rs",
"glob", "glob",
"htmlescape", "htmlescape",
"ical", "ical",
@ -1705,12 +1727,12 @@ dependencies = [
"nu-json", "nu-json",
"nu-parser", "nu-parser",
"nu-path", "nu-path",
"nu-pretty-hex",
"nu-protocol", "nu-protocol",
"nu-table", "nu-table",
"nu-term-grid", "nu-term-grid",
"num 0.4.0", "num 0.4.0",
"polars", "polars",
"pretty-hex",
"quick-xml 0.22.0", "quick-xml 0.22.0",
"rand", "rand",
"rayon", "rayon",
@ -1792,6 +1814,15 @@ dependencies = [
"serde_json", "serde_json",
] ]
[[package]]
name = "nu-pretty-hex"
version = "0.41.0"
dependencies = [
"heapless 0.7.9",
"nu-ansi-term",
"rand",
]
[[package]] [[package]]
name = "nu-protocol" name = "nu-protocol"
version = "0.1.0" version = "0.1.0"
@ -2339,12 +2370,6 @@ dependencies = [
"termtree", "termtree",
] ]
[[package]]
name = "pretty-hex"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc5c99d529f0d30937f6f4b8a86d988047327bb88d04d2c4afc356de74722131"
[[package]] [[package]]
name = "pretty_assertions" name = "pretty_assertions"
version = "1.0.0" version = "1.0.0"
@ -2885,6 +2910,15 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451"
[[package]]
name = "spin"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5"
dependencies = [
"lock_api",
]
[[package]] [[package]]
name = "stable_deref_trait" name = "stable_deref_trait"
version = "1.2.0" version = "1.2.0"

1
Cargo.toml
View file

@ -29,6 +29,7 @@ nu-engine = { path="./crates/nu-engine" }
nu-json = { path="./crates/nu-json" } nu-json = { path="./crates/nu-json" }
nu-parser = { path="./crates/nu-parser" } nu-parser = { path="./crates/nu-parser" }
nu-path = { path="./crates/nu-path" } nu-path = { path="./crates/nu-path" }
nu-pretty-hex = { path = "./crates/nu-pretty-hex" }
nu-protocol = { path = "./crates/nu-protocol" } nu-protocol = { path = "./crates/nu-protocol" }
nu-plugin = { path = "./crates/nu-plugin", optional = true } nu-plugin = { path = "./crates/nu-plugin", optional = true }
nu-table = { path = "./crates/nu-table" } nu-table = { path = "./crates/nu-table" }

3
crates/nu-command/Cargo.toml
View file

@ -11,6 +11,7 @@ build = "build.rs"
nu-engine = { path = "../nu-engine" } nu-engine = { path = "../nu-engine" }
nu-json = { path = "../nu-json" } nu-json = { path = "../nu-json" }
nu-path = { path = "../nu-path" } nu-path = { path = "../nu-path" }
nu-pretty-hex = { path = "../nu-pretty-hex" }
nu-protocol = { path = "../nu-protocol" } nu-protocol = { path = "../nu-protocol" }
nu-table = { path = "../nu-table" } nu-table = { path = "../nu-table" }
nu-term-grid = { path = "../nu-term-grid" } nu-term-grid = { path = "../nu-term-grid" }
@ -55,7 +56,6 @@ trash = { version = "2.0.2", optional = true }
unicode-segmentation = "1.8.0" unicode-segmentation = "1.8.0"
uuid = { version = "0.8.2", features = ["v4"] } uuid = { version = "0.8.2", features = ["v4"] }
htmlescape = "0.3.1" htmlescape = "0.3.1"
pretty-hex = "0.2.1"
zip = { version="0.5.9", optional=true } zip = { version="0.5.9", optional=true }
lazy_static = "1.4.0" lazy_static = "1.4.0"
strip-ansi-escapes = "0.1.1" strip-ansi-escapes = "0.1.1"
@ -66,6 +66,7 @@ digest = "0.10.0"
md5 = { package = "md-5", version = "0.10.0" } md5 = { package = "md-5", version = "0.10.0" }
sha2 = "0.10.0" sha2 = "0.10.0"
base64 = "0.13.0" base64 = "0.13.0"
encoding_rs = "0.8.30"
num = { version = "0.4.0", optional = true } num = { version = "0.4.0", optional = true }
[target.'cfg(unix)'.dependencies] [target.'cfg(unix)'.dependencies]

2
crates/nu-command/src/core_commands/echo.rs
View file

@ -34,7 +34,7 @@ impl Command for Echo {
let n = to_be_echoed.len(); let n = to_be_echoed.len();
match n.cmp(&1usize) { match n.cmp(&1usize) {
// More than one value is converted in a stream of values // More than one value is converted in a stream of values
std::cmp::Ordering::Greater => PipelineData::Stream( std::cmp::Ordering::Greater => PipelineData::ListStream(
ValueStream::from_stream(to_be_echoed.into_iter(), engine_state.ctrlc.clone()), ValueStream::from_stream(to_be_echoed.into_iter(), engine_state.ctrlc.clone()),
None, None,
), ),

1
crates/nu-command/src/default_context.rs
View file

@ -111,6 +111,7 @@ pub fn create_default_context() -> EngineState {
bind_command! { bind_command! {
BuildString, BuildString,
Char, Char,
Decode,
Format, Format,
Parse, Parse,
Size, Size,

4
crates/nu-command/src/filters/columns.rs
View file

@ -72,7 +72,7 @@ fn getcol(
.map(move |x| Value::String { val: x, span }) .map(move |x| Value::String { val: x, span })
.into_pipeline_data(engine_state.ctrlc.clone())) .into_pipeline_data(engine_state.ctrlc.clone()))
} }
PipelineData::Stream(stream, ..) => { PipelineData::ListStream(stream, ..) => {
let v: Vec<_> = stream.into_iter().collect(); let v: Vec<_> = stream.into_iter().collect();
let input_cols = get_input_cols(v); let input_cols = get_input_cols(v);
@ -81,7 +81,7 @@ fn getcol(
.map(move |x| Value::String { val: x, span }) .map(move |x| Value::String { val: x, span })
.into_pipeline_data(engine_state.ctrlc.clone())) .into_pipeline_data(engine_state.ctrlc.clone()))
} }
PipelineData::Value(_v, ..) => { PipelineData::Value(..) | PipelineData::StringStream(..) | PipelineData::ByteStream(..) => {
let cols = vec![]; let cols = vec![];
let vals = vec![]; let vals = vec![];
Ok(Value::Record { cols, vals, span }.into_pipeline_data()) Ok(Value::Record { cols, vals, span }.into_pipeline_data())

3
crates/nu-command/src/filters/drop/column.rs
View file

@ -86,7 +86,7 @@ fn dropcol(
.into_iter() .into_iter()
.into_pipeline_data(engine_state.ctrlc.clone())) .into_pipeline_data(engine_state.ctrlc.clone()))
} }
PipelineData::Stream(stream, ..) => { PipelineData::ListStream(stream, ..) => {
let mut output = vec![]; let mut output = vec![];
let v: Vec<_> = stream.into_iter().collect(); let v: Vec<_> = stream.into_iter().collect();
@ -123,6 +123,7 @@ fn dropcol(
Ok(Value::Record { cols, vals, span }.into_pipeline_data()) Ok(Value::Record { cols, vals, span }.into_pipeline_data())
} }
x => Ok(x),
} }
} }

75
crates/nu-command/src/filters/each.rs
View file

@ -76,7 +76,7 @@ impl Command for Each {
match input { match input {
PipelineData::Value(Value::Range { .. }, ..) PipelineData::Value(Value::Range { .. }, ..)
| PipelineData::Value(Value::List { .. }, ..) | PipelineData::Value(Value::List { .. }, ..)
| PipelineData::Stream { .. } => Ok(input | PipelineData::ListStream { .. } => Ok(input
.into_iter() .into_iter()
.enumerate() .enumerate()
.map(move |(idx, x)| { .map(move |(idx, x)| {
@ -109,6 +109,79 @@ impl Command for Each {
} }
}) })
.into_pipeline_data(ctrlc)), .into_pipeline_data(ctrlc)),
PipelineData::ByteStream(stream, ..) => Ok(stream
.into_iter()
.enumerate()
.map(move |(idx, x)| {
let x = Value::Binary { val: x, span };
if let Some(var) = block.signature.get_positional(0) {
if let Some(var_id) = &var.var_id {
if numbered {
stack.add_var(
*var_id,
Value::Record {
cols: vec!["index".into(), "item".into()],
vals: vec![
Value::Int {
val: idx as i64,
span,
},
x,
],
span,
},
);
} else {
stack.add_var(*var_id, x);
}
}
}
match eval_block(&engine_state, &mut stack, &block, PipelineData::new(span)) {
Ok(v) => v.into_value(span),
Err(error) => Value::Error { error },
}
})
.into_pipeline_data(ctrlc)),
PipelineData::StringStream(stream, ..) => Ok(stream
.into_iter()
.enumerate()
.map(move |(idx, x)| {
let x = match x {
Ok(x) => Value::String { val: x, span },
Err(err) => return Value::Error { error: err },
};
if let Some(var) = block.signature.get_positional(0) {
if let Some(var_id) = &var.var_id {
if numbered {
stack.add_var(
*var_id,
Value::Record {
cols: vec!["index".into(), "item".into()],
vals: vec![
Value::Int {
val: idx as i64,
span,
},
x,
],
span,
},
);
} else {
stack.add_var(*var_id, x);
}
}
}
match eval_block(&engine_state, &mut stack, &block, PipelineData::new(span)) {
Ok(v) => v.into_value(span),
Err(error) => Value::Error { error },
}
})
.into_pipeline_data(ctrlc)),
PipelineData::Value(Value::Record { cols, vals, .. }, ..) => { PipelineData::Value(Value::Record { cols, vals, .. }, ..) => {
let mut output_cols = vec![]; let mut output_cols = vec![];
let mut output_vals = vec![]; let mut output_vals = vec![];

50
crates/nu-command/src/filters/lines.rs
View file

@ -27,10 +27,11 @@ impl Command for Lines {
fn run( fn run(
&self, &self,
engine_state: &EngineState, engine_state: &EngineState,
_stack: &mut Stack, stack: &mut Stack,
call: &Call, call: &Call,
input: PipelineData, input: PipelineData,
) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> { ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
let head = call.head;
let skip_empty = call.has_flag("skip-emtpy"); let skip_empty = call.has_flag("skip-emtpy");
match input { match input {
#[allow(clippy::needless_collect)] #[allow(clippy::needless_collect)]
@ -53,7 +54,7 @@ impl Command for Lines {
Ok(iter.into_pipeline_data(engine_state.ctrlc.clone())) Ok(iter.into_pipeline_data(engine_state.ctrlc.clone()))
} }
PipelineData::Stream(stream, ..) => { PipelineData::ListStream(stream, ..) => {
let iter = stream let iter = stream
.into_iter() .into_iter()
.filter_map(move |value| { .filter_map(move |value| {
@ -81,10 +82,55 @@ impl Command for Lines {
Ok(iter.into_pipeline_data(engine_state.ctrlc.clone())) Ok(iter.into_pipeline_data(engine_state.ctrlc.clone()))
} }
PipelineData::StringStream(stream, span, ..) => {
let iter = stream
.into_iter()
.map(move |value| match value {
Ok(value) => value
.split(SPLIT_CHAR)
.filter_map(|s| {
if !s.is_empty() {
Some(Value::String {
val: s.into(),
span,
})
} else {
None
}
})
.collect::<Vec<Value>>(),
Err(err) => vec![Value::Error { error: err }],
})
.flatten();
Ok(iter.into_pipeline_data(engine_state.ctrlc.clone()))
}
PipelineData::Value(val, ..) => Err(ShellError::UnsupportedInput( PipelineData::Value(val, ..) => Err(ShellError::UnsupportedInput(
format!("Not supported input: {}", val.as_string()?), format!("Not supported input: {}", val.as_string()?),
call.head, call.head,
)), )),
PipelineData::ByteStream(..) => {
let config = stack.get_config()?;
//FIXME: Make sure this can fail in the future to let the user
//know to use a different encoding
let s = input.collect_string("", &config)?;
let lines = s
.split(SPLIT_CHAR)
.map(|s| s.to_string())
.collect::<Vec<String>>();
let iter = lines.into_iter().filter_map(move |s| {
if skip_empty && s.is_empty() {
None
} else {
Some(Value::string(s, head))
}
});
Ok(iter.into_pipeline_data(engine_state.ctrlc.clone()))
}
} }
} }
} }

87
crates/nu-command/src/filters/par_each.rs
View file

@ -139,7 +139,7 @@ impl Command for ParEach {
.into_iter() .into_iter()
.flatten() .flatten()
.into_pipeline_data(ctrlc)), .into_pipeline_data(ctrlc)),
PipelineData::Stream(stream, ..) => Ok(stream PipelineData::ListStream(stream, ..) => Ok(stream
.enumerate() .enumerate()
.par_bridge() .par_bridge()
.map(move |(idx, x)| { .map(move |(idx, x)| {
@ -179,6 +179,91 @@ impl Command for ParEach {
.into_iter() .into_iter()
.flatten() .flatten()
.into_pipeline_data(ctrlc)), .into_pipeline_data(ctrlc)),
PipelineData::StringStream(stream, ..) => Ok(stream
.enumerate()
.par_bridge()
.map(move |(idx, x)| {
let x = match x {
Ok(x) => Value::String { val: x, span },
Err(err) => return Value::Error { error: err }.into_pipeline_data(),
};
let block = engine_state.get_block(block_id);
let mut stack = stack.clone();
if let Some(var) = block.signature.get_positional(0) {
if let Some(var_id) = &var.var_id {
if numbered {
stack.add_var(
*var_id,
Value::Record {
cols: vec!["index".into(), "item".into()],
vals: vec![
Value::Int {
val: idx as i64,
span,
},
x,
],
span,
},
);
} else {
stack.add_var(*var_id, x);
}
}
}
match eval_block(&engine_state, &mut stack, block, PipelineData::new(span)) {
Ok(v) => v,
Err(error) => Value::Error { error }.into_pipeline_data(),
}
})
.collect::<Vec<_>>()
.into_iter()
.flatten()
.into_pipeline_data(ctrlc)),
PipelineData::ByteStream(stream, ..) => Ok(stream
.enumerate()
.par_bridge()
.map(move |(idx, x)| {
let x = Value::Binary { val: x, span };
let block = engine_state.get_block(block_id);
let mut stack = stack.clone();
if let Some(var) = block.signature.get_positional(0) {
if let Some(var_id) = &var.var_id {
if numbered {
stack.add_var(
*var_id,
Value::Record {
cols: vec!["index".into(), "item".into()],
vals: vec![
Value::Int {
val: idx as i64,
span,
},
x,
],
span,
},
);
} else {
stack.add_var(*var_id, x);
}
}
}
match eval_block(&engine_state, &mut stack, block, PipelineData::new(span)) {
Ok(v) => v,
Err(error) => Value::Error { error }.into_pipeline_data(),
}
})
.collect::<Vec<_>>()
.into_iter()
.flatten()
.into_pipeline_data(ctrlc)),
PipelineData::Value(Value::Record { cols, vals, .. }, ..) => { PipelineData::Value(Value::Record { cols, vals, .. }, ..) => {
let mut output_cols = vec![]; let mut output_cols = vec![];
let mut output_vals = vec![]; let mut output_vals = vec![];

3
crates/nu-command/src/filters/reject.rs
View file

@ -82,7 +82,7 @@ fn reject(
.into_iter() .into_iter()
.into_pipeline_data(engine_state.ctrlc.clone())) .into_pipeline_data(engine_state.ctrlc.clone()))
} }
PipelineData::Stream(stream, ..) => { PipelineData::ListStream(stream, ..) => {
let mut output = vec![]; let mut output = vec![];
let v: Vec<_> = stream.into_iter().collect(); let v: Vec<_> = stream.into_iter().collect();
@ -119,6 +119,7 @@ fn reject(
Ok(Value::Record { cols, vals, span }.into_pipeline_data()) Ok(Value::Record { cols, vals, span }.into_pipeline_data())
} }
x => Ok(x),
} }
} }

3
crates/nu-command/src/filters/select.rs
View file

@ -95,7 +95,7 @@ fn select(
.into_iter() .into_iter()
.into_pipeline_data(engine_state.ctrlc.clone())) .into_pipeline_data(engine_state.ctrlc.clone()))
} }
PipelineData::Stream(stream, ..) => Ok(stream PipelineData::ListStream(stream, ..) => Ok(stream
.map(move |x| { .map(move |x| {
let mut cols = vec![]; let mut cols = vec![];
let mut vals = vec![]; let mut vals = vec![];
@ -130,6 +130,7 @@ fn select(
Ok(Value::Record { cols, vals, span }.into_pipeline_data()) Ok(Value::Record { cols, vals, span }.into_pipeline_data())
} }
_ => Ok(PipelineData::new(span)),
} }
} }

12
crates/nu-command/src/filters/wrap.rs
View file

@ -43,13 +43,23 @@ impl Command for Wrap {
span, span,
}) })
.into_pipeline_data(engine_state.ctrlc.clone())), .into_pipeline_data(engine_state.ctrlc.clone())),
PipelineData::Stream(stream, ..) => Ok(stream PipelineData::ListStream(stream, ..) => Ok(stream
.map(move |x| Value::Record { .map(move |x| Value::Record {
cols: vec![name.clone()], cols: vec![name.clone()],
vals: vec![x], vals: vec![x],
span, span,
}) })
.into_pipeline_data(engine_state.ctrlc.clone())), .into_pipeline_data(engine_state.ctrlc.clone())),
PipelineData::StringStream(stream, ..) => Ok(Value::String {
val: stream.into_string("")?,
span,
}
.into_pipeline_data()),
PipelineData::ByteStream(stream, ..) => Ok(Value::Binary {
val: stream.into_vec(),
span,
}
.into_pipeline_data()),
PipelineData::Value(input, ..) => Ok(Value::Record { PipelineData::Value(input, ..) => Ok(Value::Record {
cols: vec![name], cols: vec![name],
vals: vec![input], vals: vec![input],

2
crates/nu-command/src/formats/from/delimited.rs
View file

@ -52,7 +52,7 @@ pub fn from_delimited_data(
name: Span, name: Span,
config: &Config, config: &Config,
) -> Result<PipelineData, ShellError> { ) -> Result<PipelineData, ShellError> {
let concat_string = input.collect_string("", config); let concat_string = input.collect_string("", config)?;
Ok( Ok(
from_delimited_string_to_value(concat_string, noheaders, sep, name) from_delimited_string_to_value(concat_string, noheaders, sep, name)

2
crates/nu-command/src/formats/from/eml.rs
View file

@ -183,7 +183,7 @@ fn from_eml(
head: Span, head: Span,
config: &Config, config: &Config,
) -> Result<PipelineData, ShellError> { ) -> Result<PipelineData, ShellError> {
let value = input.collect_string("", config); let value = input.collect_string("", config)?;
let body_preview = preview_body let body_preview = preview_body
.map(|b| b.item as usize) .map(|b| b.item as usize)

2
crates/nu-command/src/formats/from/ics.rs
View file

@ -93,7 +93,7 @@ END:VCALENDAR' | from ics",
} }
fn from_ics(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> { fn from_ics(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> {
let input_string = input.collect_string("", config); let input_string = input.collect_string("", config)?;
let input_bytes = input_string.as_bytes(); let input_bytes = input_string.as_bytes();
let buf_reader = BufReader::new(input_bytes); let buf_reader = BufReader::new(input_bytes);
let parser = ical::IcalParser::new(buf_reader); let parser = ical::IcalParser::new(buf_reader);

2
crates/nu-command/src/formats/from/ini.rs
View file

@ -88,7 +88,7 @@ pub fn from_ini_string_to_value(s: String, span: Span) -> Result<Value, ShellErr
} }
fn from_ini(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> { fn from_ini(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> {
let concat_string = input.collect_string("", config); let concat_string = input.collect_string("", config)?;
match from_ini_string_to_value(concat_string, head) { match from_ini_string_to_value(concat_string, head) {
Ok(x) => Ok(x.into_pipeline_data()), Ok(x) => Ok(x.into_pipeline_data()),

2
crates/nu-command/src/formats/from/json.rs
View file

@ -76,7 +76,7 @@ impl Command for FromJson {
) -> Result<nu_protocol::PipelineData, ShellError> { ) -> Result<nu_protocol::PipelineData, ShellError> {
let span = call.head; let span = call.head;
let config = stack.get_config().unwrap_or_default(); let config = stack.get_config().unwrap_or_default();
let mut string_input = input.collect_string("", &config); let mut string_input = input.collect_string("", &config)?;
string_input.push('\n'); string_input.push('\n');
// TODO: turn this into a structured underline of the nu_json error // TODO: turn this into a structured underline of the nu_json error

2
crates/nu-command/src/formats/from/ssv.rs
View file

@ -275,7 +275,7 @@ fn from_ssv(
let minimum_spaces: Option<Spanned<usize>> = let minimum_spaces: Option<Spanned<usize>> =
call.get_flag(engine_state, stack, "minimum-spaces")?; call.get_flag(engine_state, stack, "minimum-spaces")?;
let concat_string = input.collect_string("", &config); let concat_string = input.collect_string("", &config)?;
let split_at = match minimum_spaces { let split_at = match minimum_spaces {
Some(number) => number.item, Some(number) => number.item,
None => DEFAULT_MINIMUM_SPACES, None => DEFAULT_MINIMUM_SPACES,

2
crates/nu-command/src/formats/from/toml.rs
View file

@ -74,7 +74,7 @@ b = [1, 2]' | from toml",
) -> Result<nu_protocol::PipelineData, ShellError> { ) -> Result<nu_protocol::PipelineData, ShellError> {
let span = call.head; let span = call.head;
let config = stack.get_config().unwrap_or_default(); let config = stack.get_config().unwrap_or_default();
let mut string_input = input.collect_string("", &config); let mut string_input = input.collect_string("", &config)?;
string_input.push('\n'); string_input.push('\n');
Ok(convert_string_to_value(string_input, span)?.into_pipeline_data()) Ok(convert_string_to_value(string_input, span)?.into_pipeline_data())
} }

2
crates/nu-command/src/formats/from/url.rs
View file

@ -54,7 +54,7 @@ impl Command for FromUrl {
} }
fn from_url(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> { fn from_url(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> {
let concat_string = input.collect_string("", config); let concat_string = input.collect_string("", config)?;
let result = serde_urlencoded::from_str::<Vec<(String, String)>>(&concat_string); let result = serde_urlencoded::from_str::<Vec<(String, String)>>(&concat_string);

2
crates/nu-command/src/formats/from/vcf.rs
View file

@ -124,7 +124,7 @@ END:VCARD' | from vcf",
} }
fn from_vcf(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> { fn from_vcf(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> {
let input_string = input.collect_string("", config); let input_string = input.collect_string("", config)?;
let input_bytes = input_string.as_bytes(); let input_bytes = input_string.as_bytes();
let cursor = std::io::Cursor::new(input_bytes); let cursor = std::io::Cursor::new(input_bytes);
let parser = ical::VcardParser::new(cursor); let parser = ical::VcardParser::new(cursor);

2
crates/nu-command/src/formats/from/xml.rs
View file

@ -179,7 +179,7 @@ pub fn from_xml_string_to_value(s: String, span: Span) -> Result<Value, roxmltre
} }
fn from_xml(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> { fn from_xml(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> {
let concat_string = input.collect_string("", config); let concat_string = input.collect_string("", config)?;
match from_xml_string_to_value(concat_string, head) { match from_xml_string_to_value(concat_string, head) {
Ok(x) => Ok(x.into_pipeline_data()), Ok(x) => Ok(x.into_pipeline_data()),

2
crates/nu-command/src/formats/from/yaml.rs
View file

@ -206,7 +206,7 @@ pub fn from_yaml_string_to_value(s: String, span: Span) -> Result<Value, ShellEr
} }
fn from_yaml(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> { fn from_yaml(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> {
let concat_string = input.collect_string("", config); let concat_string = input.collect_string("", config)?;
match from_yaml_string_to_value(concat_string, head) { match from_yaml_string_to_value(concat_string, head) {
Ok(x) => Ok(x.into_pipeline_data()), Ok(x) => Ok(x.into_pipeline_data()),

2
crates/nu-command/src/formats/to/html.rs
View file

@ -444,7 +444,7 @@ fn html_value(value: Value, config: &Config) -> String {
let mut output_string = String::new(); let mut output_string = String::new();
match value { match value {
Value::Binary { val, .. } => { Value::Binary { val, .. } => {
let output = pretty_hex::pretty_hex(&val); let output = nu_pretty_hex::pretty_hex(&val);
output_string.push_str("<pre>"); output_string.push_str("<pre>");
output_string.push_str(&output); output_string.push_str(&output);
output_string.push_str("</pre>"); output_string.push_str("</pre>");

6
crates/nu-command/src/math/utils.rs
View file

@ -62,7 +62,7 @@ pub fn calculate(
mf: impl Fn(&[Value], &Span) -> Result<Value, ShellError>, mf: impl Fn(&[Value], &Span) -> Result<Value, ShellError>,
) -> Result<Value, ShellError> { ) -> Result<Value, ShellError> {
match values { match values {
PipelineData::Stream(s, ..) => helper_for_tables(&s.collect::<Vec<Value>>(), name, mf), PipelineData::ListStream(s, ..) => helper_for_tables(&s.collect::<Vec<Value>>(), name, mf),
PipelineData::Value(Value::List { ref vals, .. }, ..) => match &vals[..] { PipelineData::Value(Value::List { ref vals, .. }, ..) => match &vals[..] {
[Value::Record { .. }, _end @ ..] => helper_for_tables(vals, name, mf), [Value::Record { .. }, _end @ ..] => helper_for_tables(vals, name, mf),
_ => mf(vals, &name), _ => mf(vals, &name),
@ -88,5 +88,9 @@ pub fn calculate(
mf(&new_vals?, &name) mf(&new_vals?, &name)
} }
PipelineData::Value(val, ..) => mf(&[val], &name), PipelineData::Value(val, ..) => mf(&[val], &name),
_ => Err(ShellError::UnsupportedInput(
"Input data is not supported by this command.".to_string(),
name,
)),
} }
} }

6
crates/nu-command/src/path/join.rs
View file

@ -71,13 +71,17 @@ the output of 'path parse' and 'path split' subcommands."#
PipelineData::Value(val, md) => { PipelineData::Value(val, md) => {
Ok(PipelineData::Value(handle_value(val, &args, head), md)) Ok(PipelineData::Value(handle_value(val, &args, head), md))
} }
PipelineData::Stream(stream, md) => Ok(PipelineData::Stream( PipelineData::ListStream(stream, md) => Ok(PipelineData::ListStream(
ValueStream::from_stream( ValueStream::from_stream(
stream.map(move |val| handle_value(val, &args, head)), stream.map(move |val| handle_value(val, &args, head)),
engine_state.ctrlc.clone(), engine_state.ctrlc.clone(),
), ),
md, md,
)), )),
_ => Err(ShellError::UnsupportedInput(
"Input data is not supported by this command.".to_string(),
head,
)),
} }
} }

2
crates/nu-command/src/random/dice.rs
View file

@ -79,7 +79,7 @@ fn dice(
} }
}); });
Ok(PipelineData::Stream( Ok(PipelineData::ListStream(
ValueStream::from_stream(iter, engine_state.ctrlc.clone()), ValueStream::from_stream(iter, engine_state.ctrlc.clone()),
None, None,
)) ))

107
crates/nu-command/src/strings/decode.rs Normal file
View file

@ -0,0 +1,107 @@
use encoding_rs::Encoding;
use nu_engine::CallExt;
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{
Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Spanned, SyntaxShape,
Value,
};
#[derive(Clone)]
pub struct Decode;
impl Command for Decode {
fn name(&self) -> &str {
"decode"
}
fn usage(&self) -> &str {
"Decode bytes as a string."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build("decode")
.required("encoding", SyntaxShape::String, "the text encoding to use")
.category(Category::Strings)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Decode the output of an external command",
example: "cat myfile.q | decode utf-8",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let head = call.head;
let encoding: Spanned<String> = call.req(engine_state, stack, 0)?;
match input {
PipelineData::ByteStream(stream, ..) => {
let bytes: Vec<u8> = stream.flatten().collect();
let encoding = match Encoding::for_label(encoding.item.as_bytes()) {
None => Err(ShellError::SpannedLabeledError(
format!(
r#"{} is not a valid encoding, refer to https://docs.rs/encoding_rs/0.8.23/encoding_rs/#statics for a valid list of encodings"#,
encoding.item
),
"invalid encoding".into(),
encoding.span,
)),
Some(encoding) => Ok(encoding),
}?;
let result = encoding.decode(&bytes);
Ok(Value::String {
val: result.0.to_string(),
span: head,
}
.into_pipeline_data())
}
PipelineData::Value(Value::Binary { val: bytes, .. }, ..) => {
let encoding = match Encoding::for_label(encoding.item.as_bytes()) {
None => Err(ShellError::SpannedLabeledError(
format!(
r#"{} is not a valid encoding, refer to https://docs.rs/encoding_rs/0.8.23/encoding_rs/#statics for a valid list of encodings"#,
encoding.item
),
"invalid encoding".into(),
encoding.span,
)),
Some(encoding) => Ok(encoding),
}?;
let result = encoding.decode(&bytes);
Ok(Value::String {
val: result.0.to_string(),
span: head,
}
.into_pipeline_data())
}
_ => Err(ShellError::UnsupportedInput(
"non-binary input".into(),
head,
)),
}
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_examples() {
crate::test_examples(Decode)
}
}

2
crates/nu-command/src/strings/format/command.rs
View file

@ -151,7 +151,7 @@ fn format(
} }
} }
Ok(PipelineData::Stream( Ok(PipelineData::ListStream(
ValueStream::from_stream(list.into_iter(), None), ValueStream::from_stream(list.into_iter(), None),
None, None,
)) ))

2
crates/nu-command/src/strings/mod.rs
View file

@ -1,5 +1,6 @@
mod build_string; mod build_string;
mod char_; mod char_;
mod decode;
mod format; mod format;
mod parse; mod parse;
mod size; mod size;
@ -8,6 +9,7 @@ mod str_;
pub use build_string::BuildString; pub use build_string::BuildString;
pub use char_::Char; pub use char_::Char;
pub use decode::*;
pub use format::*; pub use format::*;
pub use parse::*; pub use parse::*;
pub use size::Size; pub use size::Size;

2
crates/nu-command/src/strings/parse.rs
View file

@ -126,7 +126,7 @@ fn operate(
} }
} }
Ok(PipelineData::Stream( Ok(PipelineData::ListStream(
ValueStream::from_stream(parsed.into_iter(), ctrlc), ValueStream::from_stream(parsed.into_iter(), ctrlc),
None, None,
)) ))

55
crates/nu-command/src/system/run_external.rs
View file

@ -1,4 +1,3 @@
use std::borrow::Cow;
use std::collections::HashMap; use std::collections::HashMap;
use std::env; use std::env;
use std::io::{BufRead, BufReader, Write}; use std::io::{BufRead, BufReader, Write};
@ -10,14 +9,14 @@ use std::sync::mpsc;
use nu_engine::env_to_strings; use nu_engine::env_to_strings;
use nu_protocol::engine::{EngineState, Stack}; use nu_protocol::engine::{EngineState, Stack};
use nu_protocol::{ast::Call, engine::Command, ShellError, Signature, SyntaxShape, Value}; use nu_protocol::{ast::Call, engine::Command, ShellError, Signature, SyntaxShape, Value};
use nu_protocol::{Category, Config, IntoInterruptiblePipelineData, PipelineData, Span, Spanned}; use nu_protocol::{ByteStream, Category, Config, PipelineData, Spanned};
use itertools::Itertools; use itertools::Itertools;
use nu_engine::CallExt; use nu_engine::CallExt;
use regex::Regex; use regex::Regex;
const OUTPUT_BUFFER_SIZE: usize = 8192; const OUTPUT_BUFFER_SIZE: usize = 1024;
#[derive(Clone)] #[derive(Clone)]
pub struct External; pub struct External;
@ -137,6 +136,7 @@ impl<'call> ExternalCommand<'call> {
config: Config, config: Config,
) -> Result<PipelineData, ShellError> { ) -> Result<PipelineData, ShellError> {
let mut process = self.create_command(); let mut process = self.create_command();
let head = self.name.span;
let ctrlc = engine_state.ctrlc.clone(); let ctrlc = engine_state.ctrlc.clone();
@ -223,11 +223,7 @@ impl<'call> ExternalCommand<'call> {
// from bytes to String. If no replacements are required, then the // from bytes to String. If no replacements are required, then the
// borrowed value is a proper UTF-8 string. The Owned option represents // borrowed value is a proper UTF-8 string. The Owned option represents
// a string where the values had to be replaced, thus marking it as bytes // a string where the values had to be replaced, thus marking it as bytes
let data = match String::from_utf8_lossy(bytes) { let bytes = bytes.to_vec();
Cow::Borrowed(s) => Data::String(s.into()),
Cow::Owned(_) => Data::Bytes(bytes.to_vec()),
};
let length = bytes.len(); let length = bytes.len();
buf_read.consume(length); buf_read.consume(length);
@ -237,7 +233,7 @@ impl<'call> ExternalCommand<'call> {
} }
} }
match tx.send(data) { match tx.send(bytes) {
Ok(_) => continue, Ok(_) => continue,
Err(_) => break, Err(_) => break,
} }
@ -249,11 +245,16 @@ impl<'call> ExternalCommand<'call> {
Ok(_) => Ok(()), Ok(_) => Ok(()),
} }
}); });
// The ValueStream is consumed by the next expression in the pipeline let receiver = ChannelReceiver::new(rx);
let value =
ChannelReceiver::new(rx, self.name.span).into_pipeline_data(output_ctrlc);
Ok(value) Ok(PipelineData::ByteStream(
ByteStream {
stream: Box::new(receiver),
ctrlc: output_ctrlc,
},
head,
None,
))
} }
} }
} }
@ -345,42 +346,24 @@ fn trim_enclosing_quotes(input: &str) -> String {
} }
} }
// The piped data from stdout from the external command can be either String
// or binary. We use this enum to pass the data from the spawned process
#[derive(Debug)]
enum Data {
String(String),
Bytes(Vec<u8>),
}
// Receiver used for the ValueStream // Receiver used for the ValueStream
// It implements iterator so it can be used as a ValueStream // It implements iterator so it can be used as a ValueStream
struct ChannelReceiver { struct ChannelReceiver {
rx: mpsc::Receiver<Data>, rx: mpsc::Receiver<Vec<u8>>,
span: Span,
} }
impl ChannelReceiver { impl ChannelReceiver {
pub fn new(rx: mpsc::Receiver<Data>, span: Span) -> Self { pub fn new(rx: mpsc::Receiver<Vec<u8>>) -> Self {
Self { rx, span } Self { rx }
} }
} }
impl Iterator for ChannelReceiver { impl Iterator for ChannelReceiver {
type Item = Value; type Item = Vec<u8>;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
match self.rx.recv() { match self.rx.recv() {
Ok(v) => match v { Ok(v) => Some(v),
Data::String(s) => Some(Value::String {
val: s,
span: self.span,
}),
Data::Bytes(b) => Some(Value::Binary {
val: b,
span: self.span,
}),
},
Err(_) => None, Err(_) => None,
} }
} }

2
crates/nu-command/src/viewers/griddle.rs
View file

@ -86,7 +86,7 @@ prints out the list properly."#
Ok(PipelineData::new(call.head)) Ok(PipelineData::new(call.head))
} }
} }
PipelineData::Stream(stream, ..) => { PipelineData::ListStream(stream, ..) => {
// dbg!("value::stream"); // dbg!("value::stream");
let data = convert_to_list(stream, &config, call.head); let data = convert_to_list(stream, &config, call.head);
if let Some(items) = data { if let Some(items) = data {

42
crates/nu-command/src/viewers/table.rs
View file

@ -4,8 +4,8 @@ use nu_engine::{env_to_string, CallExt};
use nu_protocol::ast::{Call, PathMember}; use nu_protocol::ast::{Call, PathMember};
use nu_protocol::engine::{Command, EngineState, Stack}; use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{ use nu_protocol::{
Category, Config, DataSource, IntoInterruptiblePipelineData, IntoPipelineData, PipelineData, Category, Config, DataSource, IntoPipelineData, PipelineData, PipelineMetadata, ShellError,
PipelineMetadata, ShellError, Signature, Span, SyntaxShape, Value, ValueStream, Signature, Span, StringStream, SyntaxShape, Value, ValueStream,
}; };
use nu_table::{StyledString, TextStyle, Theme}; use nu_table::{StyledString, TextStyle, Theme};
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
@ -47,6 +47,7 @@ impl Command for Table {
call: &Call, call: &Call,
input: PipelineData, input: PipelineData,
) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> { ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
let head = call.head;
let ctrlc = engine_state.ctrlc.clone(); let ctrlc = engine_state.ctrlc.clone();
let config = stack.get_config().unwrap_or_default(); let config = stack.get_config().unwrap_or_default();
let color_hm = get_color_config(&config); let color_hm = get_color_config(&config);
@ -60,6 +61,20 @@ impl Command for Table {
}; };
match input { match input {
PipelineData::ByteStream(stream, ..) => Ok(PipelineData::StringStream(
StringStream::from_stream(
stream.map(move |x| {
Ok(if x.iter().all(|x| x.is_ascii()) {
format!("{}", String::from_utf8_lossy(&x))
} else {
format!("{}\n", nu_pretty_hex::pretty_hex(&x))
})
}),
ctrlc,
),
head,
None,
)),
PipelineData::Value(Value::List { vals, .. }, ..) => { PipelineData::Value(Value::List { vals, .. }, ..) => {
let table = convert_to_table(row_offset, &vals, ctrlc, &config, call.head)?; let table = convert_to_table(row_offset, &vals, ctrlc, &config, call.head)?;
@ -75,7 +90,7 @@ impl Command for Table {
Ok(PipelineData::new(call.head)) Ok(PipelineData::new(call.head))
} }
} }
PipelineData::Stream(stream, metadata) => { PipelineData::ListStream(stream, metadata) => {
let stream = match metadata { let stream = match metadata {
Some(PipelineMetadata { Some(PipelineMetadata {
data_source: DataSource::Ls, data_source: DataSource::Ls,
@ -161,14 +176,20 @@ impl Command for Table {
let head = call.head; let head = call.head;
Ok(PagingTableCreator { Ok(PipelineData::StringStream(
StringStream::from_stream(
PagingTableCreator {
row_offset, row_offset,
config, config,
ctrlc: ctrlc.clone(), ctrlc: ctrlc.clone(),
head, head,
stream, stream,
} },
.into_pipeline_data(ctrlc)) ctrlc,
),
head,
None,
))
} }
PipelineData::Value(Value::Record { cols, vals, .. }, ..) => { PipelineData::Value(Value::Record { cols, vals, .. }, ..) => {
let mut output = vec![]; let mut output = vec![];
@ -363,7 +384,7 @@ struct PagingTableCreator {
} }
impl Iterator for PagingTableCreator { impl Iterator for PagingTableCreator {
type Item = Value; type Item = Result<String, ShellError>;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
let mut batch = vec![]; let mut batch = vec![];
@ -418,12 +439,9 @@ impl Iterator for PagingTableCreator {
Ok(Some(table)) => { Ok(Some(table)) => {
let result = nu_table::draw_table(&table, term_width, &color_hm, &self.config); let result = nu_table::draw_table(&table, term_width, &color_hm, &self.config);
Some(Value::String { Some(Ok(result))
val: result,
span: self.head,
})
} }
Err(err) => Some(Value::Error { error: err }), Err(err) => Some(Err(err)),
_ => None, _ => None,
} }
} }

2
crates/nu-engine/src/eval.rs
View file

@ -435,7 +435,7 @@ pub fn eval_subexpression(
let config = stack.get_config().unwrap_or_default(); let config = stack.get_config().unwrap_or_default();
let mut s = input.collect_string("", &config); let mut s = input.collect_string("", &config)?;
if s.ends_with('\n') { if s.ends_with('\n') {
s.pop(); s.pop();
} }

18
crates/nu-plugin/src/plugin/declaration.rs
View file

@ -72,7 +72,7 @@ impl Command for PluginDeclaration {
let input = match input { let input = match input {
PipelineData::Value(value, ..) => value, PipelineData::Value(value, ..) => value,
PipelineData::Stream(stream, ..) => { PipelineData::ListStream(stream, ..) => {
let values = stream.collect::<Vec<Value>>(); let values = stream.collect::<Vec<Value>>();
Value::List { Value::List {
@ -80,6 +80,22 @@ impl Command for PluginDeclaration {
span: call.head, span: call.head,
} }
} }
PipelineData::StringStream(stream, ..) => {
let val = stream.into_string("")?;
Value::String {
val,
span: call.head,
}
}
PipelineData::ByteStream(stream, ..) => {
let val = stream.into_vec();
Value::Binary {
val,
span: call.head,
}
}
}; };
// Create message to plugin to indicate that signature is required and // Create message to plugin to indicate that signature is required and

201
crates/nu-pretty-hex/Cargo.lock generated Normal file
View file

@ -0,0 +1,201 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "as-slice"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45403b49e3954a4b8428a0ac21a4b7afadccf92bfd96273f1a58cd4812496ae0"
dependencies = [
"generic-array 0.12.4",
"generic-array 0.13.3",
"generic-array 0.14.4",
"stable_deref_trait",
]
[[package]]
name = "byteorder"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "generic-array"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd"
dependencies = [
"typenum",
]
[[package]]
name = "generic-array"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f797e67af32588215eaaab8327027ee8e71b9dd0b2b26996aedf20c030fce309"
dependencies = [
"typenum",
]
[[package]]
name = "generic-array"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "hash32"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4041af86e63ac4298ce40e5cca669066e75b6f1aa3390fe2561ffa5e1d9f4cc"
dependencies = [
"byteorder",
]
[[package]]
name = "heapless"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "634bd4d29cbf24424d0a4bfcbf80c6960129dc24424752a7d1d1390607023422"
dependencies = [
"as-slice",
"generic-array 0.14.4",
"hash32",
"stable_deref_trait",
]
[[package]]
name = "libc"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41"
[[package]]
name = "nu-ansi-term"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bd69a141e8fdfa5ac882d8b816db2b9ad138ef7e3baa7cb753a9b3789aa8c7e"
dependencies = [
"winapi",
]
[[package]]
name = "nu-pretty-hex"
version = "0.2.1"
dependencies = [
"heapless",
"nu-ansi-term",
"rand",
]
[[package]]
name = "ppv-lite86"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
[[package]]
name = "rand"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
"rand_hc",
]
[[package]]
name = "rand_chacha"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
dependencies = [
"getrandom",
]
[[package]]
name = "rand_hc"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
dependencies = [
"rand_core",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]]
name = "typenum"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06"
[[package]]
name = "version_check"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

27
crates/nu-pretty-hex/Cargo.toml Normal file
View file

@ -0,0 +1,27 @@
[package]
authors = ["Andrei Volnin <wolandr@gmail.com>", "The Nu Project Contributors"]
description = "Pretty hex dump of bytes slice in the common style."
edition = "2018"
license = "MIT"
name = "nu-pretty-hex"
version = "0.41.0"
[lib]
doctest = false
name = "nu_pretty_hex"
path = "src/lib.rs"
[[bin]]
name = "nu_pretty_hex"
path = "src/main.rs"
[dependencies]
nu-ansi-term = "0.39.0"
rand = "0.8.3"
[dev-dependencies]
heapless = { version = "0.7.8", default-features = false }
# [features]
# default = ["alloc"]
# alloc = []

21
crates/nu-pretty-hex/LICENSE Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2018 Andrei Volnin
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

81
crates/nu-pretty-hex/README.md Normal file
View file

@ -0,0 +1,81 @@
# nu-pretty-hex
An update of prett-hex to make it prettier
[![crates.io](https://img.shields.io/crates/v/pretty-hex.svg)](https://crates.io/crates/pretty-hex)
[![docs.rs](https://docs.rs/pretty-hex/badge.svg)](https://docs.rs/pretty-hex)
A Rust library providing pretty hex dump.
A `simple_hex()` way renders one-line hex dump, a `pretty_hex()` way renders
columned multi-line hex dump with addressing and ASCII representation.
A `config_hex()` way renders hex dump in specified format.
## Inspiration
[Hexed](https://github.com/adolfohw/hexed) \
[Hexyl](https://github.com/sharkdp/hexyl) \
[Pretty-hex](https://github.com/wolandr/pretty-hex)
## Example of `simple_hex()`
```rust
use pretty_hex::*;
let v = vec![222, 173, 190, 239, 202, 254, 32, 24];
assert_eq!(simple_hex(&v), format!("{}", v.hex_dump()));
println!("{}", v.hex_dump());
```
Output:
```text
de ad be ef ca fe 20 18
```
## Example of `pretty_hex()`
```rust
use pretty_hex::*;
let v: &[u8] = &random::<[u8;30]>();
assert_eq!(pretty_hex(&v), format!("{:?}", v.hex_dump()));
println!("{:?}", v.hex_dump());
```
Output:
```text
Length: 30 (0x1e) bytes
0000: 6b 4e 1a c3 af 03 d2 1e 7e 73 ba c8 bd 84 0f 83 kN......~s......
0010: 89 d5 cf 90 23 67 4b 48 db b1 bc 35 bf ee ....#gKH...5..
```
## Example of `config_hex()`
```rust
use pretty_hex::*;
let cfg = HexConfig {title: false, width: 8, group: 0, ..HexConfig::default() };
let v = &include_bytes!("data");
assert_eq!(config_hex(&v, cfg), format!("{:?}", v.hex_conf(cfg)));
println!("{:?}", v.hex_conf(cfg));
```
Output:
```text
0000: 6b 4e 1a c3 af 03 d2 1e kN......
0008: 7e 73 ba c8 bd 84 0f 83 ~s......
0010: 89 d5 cf 90 23 67 4b 48 ....#gKH
0018: db b1 bc 35 bf ee ...5..
```
---
Inspired by [haskell's pretty-hex](https://hackage.haskell.org/package/pretty-hex-1.0).

66
crates/nu-pretty-hex/src/lib.rs Normal file
View file

@ -0,0 +1,66 @@
// #![no_std]
//! A Rust library providing pretty hex dump.
//!
//! A `simple_hex()` way renders one-line hex dump, and a `pretty_hex()` way renders
//! columned multi-line hex dump with addressing and ASCII representation.
//! A `config_hex()` way renders hex dump in specified format.
//!
//! ## Example of `simple_hex()`
//! ```
//! use pretty_hex::*;
//!
//! let v = vec![222, 173, 190, 239, 202, 254, 32, 24];
//! # #[cfg(feature = "alloc")]
//! assert_eq!(simple_hex(&v), format!("{}", v.hex_dump()));
//!
//! println!("{}", v.hex_dump());
//! ```
//! Output:
//!
//! ```text
//! de ad be ef ca fe 20 18
//! ```
//! ## Example of `pretty_hex()`
//! ```
//! use pretty_hex::*;
//!
//! let v = &include_bytes!("../tests/data");
//! # #[cfg(feature = "alloc")]
//! assert_eq!(pretty_hex(&v), format!("{:?}", v.hex_dump()));
//!
//! println!("{:?}", v.hex_dump());
//! ```
//! Output:
//!
//! ```text
//! Length: 30 (0x1e) bytes
//! 0000: 6b 4e 1a c3 af 03 d2 1e 7e 73 ba c8 bd 84 0f 83 kN......~s......
//! 0010: 89 d5 cf 90 23 67 4b 48 db b1 bc 35 bf ee ....#gKH...5..
//! ```
//! ## Example of `config_hex()`
//! ```
//! use pretty_hex::*;
//!
//! let cfg = HexConfig {title: false, width: 8, group: 0, ..HexConfig::default() };
//!
//! let v = &include_bytes!("../tests/data");
//! # #[cfg(feature = "alloc")]
//! assert_eq!(config_hex(&v, cfg), format!("{:?}", v.hex_conf(cfg)));
//!
//! println!("{:?}", v.hex_conf(cfg));
//! ```
//! Output:
//!
//! ```text
//! 0000: 6b 4e 1a c3 af 03 d2 1e kN......
//! 0008: 7e 73 ba c8 bd 84 0f 83 ~s......
//! 0010: 89 d5 cf 90 23 67 4b 48 ....#gKH
//! 0018: db b1 bc 35 bf ee ...5..
//! ```
#[cfg(feature = "alloc")]
extern crate alloc;
mod pretty_hex;
pub use pretty_hex::*;

50
crates/nu-pretty-hex/src/main.rs Normal file
View file

@ -0,0 +1,50 @@
use nu_pretty_hex::*;
fn main() {
let config = HexConfig {
title: true,
ascii: true,
width: 16,
group: 4,
chunk: 1,
skip: Some(10),
// length: Some(5),
// length: None,
length: Some(50),
};
let my_string = "Darren Schroeder 😉";
println!("ConfigHex\n{}\n", config_hex(&my_string, config));
println!("SimpleHex\n{}\n", simple_hex(&my_string));
println!("PrettyHex\n{}\n", pretty_hex(&my_string));
println!("ConfigHex\n{}\n", config_hex(&my_string, config));
// let mut my_str = String::new();
// for x in 0..256 {
// my_str.push(x as u8);
// }
let mut v: Vec<u8> = vec![];
for x in 0..=127 {
v.push(x);
}
let my_str = String::from_utf8_lossy(&v[..]);
println!("First128\n{}\n", pretty_hex(&my_str.as_bytes()));
println!(
"First128-Param\n{}\n",
config_hex(&my_str.as_bytes(), config)
);
let mut r_str = String::new();
for _ in 0..=127 {
r_str.push(rand::random::<u8>() as char);
}
println!("Random127\n{}\n", pretty_hex(&r_str));
}
//chunk 0 44617272656e20536368726f65646572 Darren Schroeder
//chunk 1 44 61 72 72 65 6e 20 53 63 68 72 6f 65 64 65 72 Darren Schroeder
//chunk 2 461 7272 656e 2053 6368 726f 6564 6572 Darren Schroeder
//chunk 3 46172 72656e 205363 68726f 656465 72 Darren Schroeder
//chunk 4 44617272 656e2053 6368726f 65646572 Darren Schroeder

299
crates/nu-pretty-hex/src/pretty_hex.rs Normal file
View file

@ -0,0 +1,299 @@
use core::primitive::str;
use core::{default::Default, fmt};
use nu_ansi_term::{Color, Style};
/// Returns a one-line hexdump of `source` grouped in default format without header
/// and ASCII column.
pub fn simple_hex<T: AsRef<[u8]>>(source: &T) -> String {
let mut writer = String::new();
hex_write(&mut writer, source, HexConfig::simple(), None).unwrap_or(());
writer
}
/// Dump `source` as hex octets in default format without header and ASCII column to the `writer`.
pub fn simple_hex_write<T, W>(writer: &mut W, source: &T) -> fmt::Result
where
T: AsRef<[u8]>,
W: fmt::Write,
{
hex_write(writer, source, HexConfig::simple(), None)
}
/// Return a multi-line hexdump in default format complete with addressing, hex digits,
/// and ASCII representation.
pub fn pretty_hex<T: AsRef<[u8]>>(source: &T) -> String {
let mut writer = String::new();
hex_write(&mut writer, source, HexConfig::default(), Some(true)).unwrap_or(());
writer
}
/// Write multi-line hexdump in default format complete with addressing, hex digits,
/// and ASCII representation to the writer.
pub fn pretty_hex_write<T, W>(writer: &mut W, source: &T) -> fmt::Result
where
T: AsRef<[u8]>,
W: fmt::Write,
{
hex_write(writer, source, HexConfig::default(), Some(true))
}
/// Return a hexdump of `source` in specified format.
pub fn config_hex<T: AsRef<[u8]>>(source: &T, cfg: HexConfig) -> String {
let mut writer = String::new();
hex_write(&mut writer, source, cfg, Some(true)).unwrap_or(());
writer
}
/// Configuration parameters for hexdump.
#[derive(Clone, Copy, Debug)]
pub struct HexConfig {
/// Write first line header with data length.
pub title: bool,
/// Append ASCII representation column.
pub ascii: bool,
/// Source bytes per row. 0 for single row without address prefix.
pub width: usize,
/// Chunks count per group. 0 for single group (column).
pub group: usize,
/// Source bytes per chunk (word). 0 for single word.
pub chunk: usize,
/// Bytes from 0 to skip
pub skip: Option<usize>,
/// Length to return
pub length: Option<usize>,
}
/// Default configuration with `title`, `ascii`, 16 source bytes `width` grouped to 4 separate
/// hex bytes. Using in `pretty_hex`, `pretty_hex_write` and `fmt::Debug` implementation.
impl Default for HexConfig {
fn default() -> HexConfig {
HexConfig {
title: true,
ascii: true,
width: 16,
group: 4,
chunk: 1,
skip: None,
length: None,
}
}
}
impl HexConfig {
/// Returns configuration for `simple_hex`, `simple_hex_write` and `fmt::Display` implementation.
pub fn simple() -> Self {
HexConfig::default().to_simple()
}
fn delimiter(&self, i: usize) -> &'static str {
if i > 0 && self.chunk > 0 && i % self.chunk == 0 {
if self.group > 0 && i % (self.group * self.chunk) == 0 {
" "
} else {
" "
}
} else {
""
}
}
fn to_simple(self) -> Self {
HexConfig {
title: false,
ascii: false,
width: 0,
..self
}
}
}
fn categorize_byte(byte: &u8) -> (Style, Option<char>) {
// This section is here so later we can configure these items
let null_char_style = Style::default().fg(Color::Fixed(242));
let null_char = Some('0');
let ascii_printable_style = Style::default().fg(Color::Cyan).bold();
let ascii_printable = None;
let ascii_space_style = Style::default().fg(Color::Green).bold();
let ascii_space = Some(' ');
let ascii_white_space_style = Style::default().fg(Color::Green).bold();
let ascii_white_space = Some('_');
let ascii_other_style = Style::default().fg(Color::Purple).bold();
let ascii_other = Some('•');
let non_ascii_style = Style::default().fg(Color::Yellow).bold();
let non_ascii = Some('×'); // or Some('.')
if byte == &0 {
(null_char_style, null_char)
} else if byte.is_ascii_graphic() {
(ascii_printable_style, ascii_printable)
} else if byte.is_ascii_whitespace() {
// 0x20 == 32 decimal - replace with a real space
if byte == &32 {
(ascii_space_style, ascii_space)
} else {
(ascii_white_space_style, ascii_white_space)
}
} else if byte.is_ascii() {
(ascii_other_style, ascii_other)
} else {
(non_ascii_style, non_ascii)
}
}
/// Write hex dump in specified format.
pub fn hex_write<T, W>(
writer: &mut W,
source: &T,
cfg: HexConfig,
with_color: Option<bool>,
) -> fmt::Result
where
T: AsRef<[u8]>,
W: fmt::Write,
{
let use_color = with_color.unwrap_or(false);
if source.as_ref().is_empty() {
return Ok(());
}
let amount = match cfg.length {
Some(len) => len,
None => source.as_ref().len(),
};
let skip = cfg.skip.unwrap_or(0);
let source_part_vec: Vec<u8> = source
.as_ref()
.iter()
.skip(skip)
.take(amount)
.map(|&x| x as u8)
.collect();
if cfg.title {
if use_color {
writeln!(
writer,
"Length: {0} (0x{0:x}) bytes | {1}printable {2}whitespace {3}ascii_other {4}non_ascii{5}",
source_part_vec.len(),
Style::default().fg(Color::Cyan).bold().prefix(),
Style::default().fg(Color::Green).bold().prefix(),
Style::default().fg(Color::Purple).bold().prefix(),
Style::default().fg(Color::Yellow).bold().prefix(),
Style::default().fg(Color::Yellow).suffix()
)?;
} else {
writeln!(writer, "Length: {0} (0x{0:x}) bytes", source_part_vec.len(),)?;
}
}
let lines = source_part_vec.chunks(if cfg.width > 0 {
cfg.width
} else {
source_part_vec.len()
});
let lines_len = lines.len();
for (i, row) in lines.enumerate() {
if cfg.width > 0 {
let style = Style::default().fg(Color::Cyan);
if use_color {
write!(
writer,
"{}{:08x}{}: ",
style.prefix(),
i * cfg.width + skip,
style.suffix()
)?;
} else {
write!(writer, "{:08x}: ", i * cfg.width + skip,)?;
}
}
for (i, x) in row.as_ref().iter().enumerate() {
if use_color {
let (style, _char) = categorize_byte(x);
write!(
writer,
"{}{}{:02x}{}",
cfg.delimiter(i),
style.prefix(),
x,
style.suffix()
)?;
} else {
write!(writer, "{}{:02x}", cfg.delimiter(i), x,)?;
}
}
if cfg.ascii {
for j in row.len()..cfg.width {
write!(writer, "{} ", cfg.delimiter(j))?;
}
write!(writer, " ")?;
for x in row {
let (style, a_char) = categorize_byte(x);
let replacement_char = match a_char {
Some(c) => c,
None => *x as char,
};
if use_color {
write!(
writer,
"{}{}{}",
style.prefix(),
replacement_char,
style.suffix()
)?;
} else {
write!(writer, "{}", replacement_char,)?;
}
}
}
if i + 1 < lines_len {
writeln!(writer)?;
}
}
Ok(())
}
/// Reference wrapper for use in arguments formatting.
pub struct Hex<'a, T: 'a>(&'a T, HexConfig);
impl<'a, T: 'a + AsRef<[u8]>> fmt::Display for Hex<'a, T> {
/// Formats the value by `simple_hex_write` using the given formatter.
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
hex_write(f, self.0, self.1.to_simple(), None)
}
}
impl<'a, T: 'a + AsRef<[u8]>> fmt::Debug for Hex<'a, T> {
/// Formats the value by `pretty_hex_write` using the given formatter.
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
hex_write(f, self.0, self.1, None)
}
}
/// Allows generates hex dumps to a formatter.
pub trait PrettyHex: Sized {
/// Wrap self reference for use in `std::fmt::Display` and `std::fmt::Debug`
/// formatting as hex dumps.
fn hex_dump(&self) -> Hex<Self>;
/// Wrap self reference for use in `std::fmt::Display` and `std::fmt::Debug`
/// formatting as hex dumps in specified format.
fn hex_conf(&self, cfg: HexConfig) -> Hex<Self>;
}
impl<T> PrettyHex for T
where
T: AsRef<[u8]>,
{
fn hex_dump(&self) -> Hex<Self> {
Hex(self, HexConfig::default())
}
fn hex_conf(&self, cfg: HexConfig) -> Hex<Self> {
Hex(self, cfg)
}
}

17
crates/nu-pretty-hex/tests/256.txt Normal file
View file

@ -0,0 +1,17 @@
Length: 256 (0x100) bytes
0000: 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f ................
0010: 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f ................
0020: 20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f !"#$%&'()*+,-./
0030: 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d 3e 3f 0123456789:;<=>?
0040: 40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f @ABCDEFGHIJKLMNO
0050: 50 51 52 53 54 55 56 57 58 59 5a 5b 5c 5d 5e 5f PQRSTUVWXYZ[\]^_
0060: 60 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 6f `abcdefghijklmno
0070: 70 71 72 73 74 75 76 77 78 79 7a 7b 7c 7d 7e 7f pqrstuvwxyz{|}~.
0080: 80 81 82 83 84 85 86 87 88 89 8a 8b 8c 8d 8e 8f ................
0090: 90 91 92 93 94 95 96 97 98 99 9a 9b 9c 9d 9e 9f ................
00a0: a0 a1 a2 a3 a4 a5 a6 a7 a8 a9 aa ab ac ad ae af ................
00b0: b0 b1 b2 b3 b4 b5 b6 b7 b8 b9 ba bb bc bd be bf ................
00c0: c0 c1 c2 c3 c4 c5 c6 c7 c8 c9 ca cb cc cd ce cf ................
00d0: d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 da db dc dd de df ................
00e0: e0 e1 e2 e3 e4 e5 e6 e7 e8 e9 ea eb ec ed ee ef ................
00f0: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff ................

1
crates/nu-pretty-hex/tests/data Normal file
View file

@ -0,0 +1 @@
kNУЏв~sКШН<D0A8><0F><>еЯ<D0B5>#gKHлБМ5Пю

175
crates/nu-pretty-hex/tests/tests.rs Normal file
View file

@ -0,0 +1,175 @@
// #![no_std]
#[cfg(feature = "alloc")]
extern crate alloc;
extern crate nu_pretty_hex;
#[cfg(feature = "alloc")]
use alloc::{format, string::String, vec, vec::Vec};
use nu_pretty_hex::*;
#[cfg(feature = "alloc")]
#[test]
fn test_simple() {
let bytes: Vec<u8> = (0..16).collect();
let expected = "00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f";
assert_eq!(expected, simple_hex(&bytes));
assert_eq!(expected, bytes.hex_dump().to_string());
assert_eq!(simple_hex(&bytes), config_hex(&bytes, HexConfig::simple()));
let mut have = String::new();
simple_hex_write(&mut have, &bytes).unwrap();
assert_eq!(expected, have);
let str = "string";
let string: String = String::from("string");
let slice: &[u8] = &[0x73, 0x74, 0x72, 0x69, 0x6e, 0x67];
assert_eq!(simple_hex(&str), "73 74 72 69 6e 67");
assert_eq!(simple_hex(&str), simple_hex(&string));
assert_eq!(simple_hex(&str), simple_hex(&slice));
assert!(simple_hex(&vec![]).is_empty());
}
#[cfg(feature = "alloc")]
#[test]
fn test_pretty() {
let bytes: Vec<u8> = (0..256).map(|x| x as u8).collect();
let want = include_str!("256.txt");
let mut hex = String::new();
pretty_hex_write(&mut hex, &bytes).unwrap();
assert_eq!(want, hex);
assert_eq!(want, format!("{:?}", bytes.hex_dump()));
assert_eq!(want, pretty_hex(&bytes));
assert_eq!(want, config_hex(&bytes, HexConfig::default()));
assert_eq!("Length: 0 (0x0) bytes\n", pretty_hex(&vec![]));
}
#[cfg(feature = "alloc")]
#[test]
fn test_config() {
let cfg = HexConfig {
title: false,
ascii: false,
width: 0,
group: 0,
chunk: 0,
};
assert!(config_hex(&vec![], cfg).is_empty());
assert_eq!("2425262728", config_hex(&"$%&'(", cfg));
let v = include_bytes!("data");
let cfg = HexConfig {
title: false,
group: 8,
..HexConfig::default()
};
let hex = "0000: 6b 4e 1a c3 af 03 d2 1e 7e 73 ba c8 bd 84 0f 83 kN......~s......\n\
0010: 89 d5 cf 90 23 67 4b 48 db b1 bc 35 bf ee ....#gKH...5..";
assert_eq!(hex, config_hex(&v, cfg));
assert_eq!(hex, format!("{:?}", v.hex_conf(cfg)));
let mut str = String::new();
hex_write(&mut str, v, cfg).unwrap();
assert_eq!(hex, str);
assert_eq!(
config_hex(
&v,
HexConfig {
ascii: false,
..cfg
}
),
"0000: 6b 4e 1a c3 af 03 d2 1e 7e 73 ba c8 bd 84 0f 83\n\
0010: 89 d5 cf 90 23 67 4b 48 db b1 bc 35 bf ee"
);
assert_eq!(
config_hex(
&v,
HexConfig {
ascii: false,
group: 4,
chunk: 2,
..cfg
}
),
"0000: 6b4e 1ac3 af03 d21e 7e73 bac8 bd84 0f83\n\
0010: 89d5 cf90 2367 4b48 dbb1 bc35 bfee"
);
let v: Vec<u8> = (0..21).collect();
let want = r##"Length: 21 (0x15) bytes
0000: 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f ................
0010: 10 11 12 13 14 ....."##;
assert_eq!(want, pretty_hex(&v));
let v: Vec<u8> = (0..13).collect();
assert_eq!(
config_hex(
&v,
HexConfig {
title: false,
ascii: true,
width: 11,
group: 2,
chunk: 3
}
),
"0000: 000102 030405 060708 090a ...........\n\
000b: 0b0c .."
);
let v: Vec<u8> = (0..19).collect();
assert_eq!(
config_hex(
&v,
HexConfig {
title: false,
ascii: true,
width: 16,
group: 3,
chunk: 3
}
),
"0000: 000102 030405 060708 090a0b 0c0d0e 0f ................\n\
0010: 101112 ..."
);
let cfg = HexConfig {
title: false,
group: 0,
..HexConfig::default()
};
assert_eq!(
format!("{:?}", v.hex_conf(cfg)),
"0000: 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f ................\n\
0010: 10 11 12 ..."
);
assert_eq!(
v.hex_conf(cfg).to_string(),
"00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12"
);
}
// This test case checks that hex_write works even without the alloc crate.
// Decorators to this function like simple_hex_write or PrettyHex::hex_dump()
// will be tested when the alloc feature is selected because it feels quite
// cumbersome to set up these tests without the comodity from `alloc`.
#[test]
fn test_hex_write_with_simple_config() {
let config = HexConfig::simple();
let bytes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
let expected =
core::str::from_utf8(b"00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f").unwrap();
// let expected =
// "\u{1b}[38;5;242m00\u{1b}[0m \u{1b}[1;35m01\u{1b}[0m \u{1b}[1;35m02\u{1b}[0m \u{1b}[1;";
let mut buffer = heapless::Vec::<u8, 50>::new();
hex_write(&mut buffer, &bytes, config, None).unwrap();
let have = core::str::from_utf8(&buffer).unwrap();
assert_eq!(expected, have);
}

123
crates/nu-protocol/src/pipeline_data.rs
View file

@ -1,6 +1,8 @@
use std::sync::{atomic::AtomicBool, Arc}; use std::sync::{atomic::AtomicBool, Arc};
use crate::{ast::PathMember, Config, ShellError, Span, Value, ValueStream}; use crate::{
ast::PathMember, ByteStream, Config, ShellError, Span, StringStream, Value, ValueStream,
};
/// The foundational abstraction for input and output to commands /// The foundational abstraction for input and output to commands
/// ///
@ -34,7 +36,9 @@ use crate::{ast::PathMember, Config, ShellError, Span, Value, ValueStream};
#[derive(Debug)] #[derive(Debug)]
pub enum PipelineData { pub enum PipelineData {
Value(Value, Option<PipelineMetadata>), Value(Value, Option<PipelineMetadata>),
Stream(ValueStream, Option<PipelineMetadata>), ListStream(ValueStream, Option<PipelineMetadata>),
StringStream(StringStream, Span, Option<PipelineMetadata>),
ByteStream(ByteStream, Span, Option<PipelineMetadata>),
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@ -54,7 +58,9 @@ impl PipelineData {
pub fn metadata(&self) -> Option<PipelineMetadata> { pub fn metadata(&self) -> Option<PipelineMetadata> {
match self { match self {
PipelineData::Stream(_, x) => x.clone(), PipelineData::ListStream(_, x) => x.clone(),
PipelineData::ByteStream(_, _, x) => x.clone(),
PipelineData::StringStream(_, _, x) => x.clone(),
PipelineData::Value(_, x) => x.clone(), PipelineData::Value(_, x) => x.clone(),
} }
} }
@ -63,27 +69,49 @@ impl PipelineData {
match self { match self {
PipelineData::Value(Value::Nothing { .. }, ..) => Value::nothing(span), PipelineData::Value(Value::Nothing { .. }, ..) => Value::nothing(span),
PipelineData::Value(v, ..) => v, PipelineData::Value(v, ..) => v,
PipelineData::Stream(s, ..) => Value::List { PipelineData::ListStream(s, ..) => Value::List {
vals: s.collect(), vals: s.collect(),
span, // FIXME? span, // FIXME?
}, },
PipelineData::StringStream(s, ..) => {
let mut output = String::new();
for item in s {
match item {
Ok(s) => output.push_str(&s),
Err(err) => return Value::Error { error: err },
}
}
Value::String {
val: output,
span, // FIXME?
}
}
PipelineData::ByteStream(s, ..) => Value::Binary {
val: s.flatten().collect(),
span, // FIXME?
},
} }
} }
pub fn into_interruptible_iter(self, ctrlc: Option<Arc<AtomicBool>>) -> PipelineIterator { pub fn into_interruptible_iter(self, ctrlc: Option<Arc<AtomicBool>>) -> PipelineIterator {
let mut iter = self.into_iter(); let mut iter = self.into_iter();
if let PipelineIterator(PipelineData::Stream(s, ..)) = &mut iter { if let PipelineIterator(PipelineData::ListStream(s, ..)) = &mut iter {
s.ctrlc = ctrlc; s.ctrlc = ctrlc;
} }
iter iter
} }
pub fn collect_string(self, separator: &str, config: &Config) -> String { pub fn collect_string(self, separator: &str, config: &Config) -> Result<String, ShellError> {
match self { match self {
PipelineData::Value(v, ..) => v.into_string(separator, config), PipelineData::Value(v, ..) => Ok(v.into_string(separator, config)),
PipelineData::Stream(s, ..) => s.into_string(separator, config), PipelineData::ListStream(s, ..) => Ok(s.into_string(separator, config)),
PipelineData::StringStream(s, ..) => s.into_string(separator),
PipelineData::ByteStream(s, ..) => {
Ok(String::from_utf8_lossy(&s.flatten().collect::<Vec<_>>()).to_string())
}
} }
} }
@ -94,12 +122,13 @@ impl PipelineData {
) -> Result<Value, ShellError> { ) -> Result<Value, ShellError> {
match self { match self {
// FIXME: there are probably better ways of doing this // FIXME: there are probably better ways of doing this
PipelineData::Stream(stream, ..) => Value::List { PipelineData::ListStream(stream, ..) => Value::List {
vals: stream.collect(), vals: stream.collect(),
span: head, span: head,
} }
.follow_cell_path(cell_path), .follow_cell_path(cell_path),
PipelineData::Value(v, ..) => v.follow_cell_path(cell_path), PipelineData::Value(v, ..) => v.follow_cell_path(cell_path),
_ => Err(ShellError::IOError("can't follow stream paths".into())),
} }
} }
@ -111,12 +140,13 @@ impl PipelineData {
) -> Result<(), ShellError> { ) -> Result<(), ShellError> {
match self { match self {
// FIXME: there are probably better ways of doing this // FIXME: there are probably better ways of doing this
PipelineData::Stream(stream, ..) => Value::List { PipelineData::ListStream(stream, ..) => Value::List {
vals: stream.collect(), vals: stream.collect(),
span: head, span: head,
} }
.update_cell_path(cell_path, callback), .update_cell_path(cell_path, callback),
PipelineData::Value(v, ..) => v.update_cell_path(cell_path, callback), PipelineData::Value(v, ..) => v.update_cell_path(cell_path, callback),
_ => Ok(()),
} }
} }
@ -134,7 +164,14 @@ impl PipelineData {
PipelineData::Value(Value::List { vals, .. }, ..) => { PipelineData::Value(Value::List { vals, .. }, ..) => {
Ok(vals.into_iter().map(f).into_pipeline_data(ctrlc)) Ok(vals.into_iter().map(f).into_pipeline_data(ctrlc))
} }
PipelineData::Stream(stream, ..) => Ok(stream.map(f).into_pipeline_data(ctrlc)), PipelineData::ListStream(stream, ..) => Ok(stream.map(f).into_pipeline_data(ctrlc)),
PipelineData::StringStream(stream, span, ..) => Ok(stream
.map(move |x| match x {
Ok(s) => f(Value::String { val: s, span }),
Err(err) => Value::Error { error: err },
})
.into_pipeline_data(ctrlc)),
PipelineData::Value(Value::Range { val, .. }, ..) => { PipelineData::Value(Value::Range { val, .. }, ..) => {
Ok(val.into_range_iter()?.map(f).into_pipeline_data(ctrlc)) Ok(val.into_range_iter()?.map(f).into_pipeline_data(ctrlc))
} }
@ -142,6 +179,11 @@ impl PipelineData {
Value::Error { error } => Err(error), Value::Error { error } => Err(error),
v => Ok(v.into_pipeline_data()), v => Ok(v.into_pipeline_data()),
}, },
PipelineData::ByteStream(_, span, ..) => Err(ShellError::UnsupportedInput(
"Binary output from this command may need to be decoded using the 'decode' command"
.into(),
span,
)),
} }
} }
@ -161,14 +203,27 @@ impl PipelineData {
PipelineData::Value(Value::List { vals, .. }, ..) => { PipelineData::Value(Value::List { vals, .. }, ..) => {
Ok(vals.into_iter().map(f).flatten().into_pipeline_data(ctrlc)) Ok(vals.into_iter().map(f).flatten().into_pipeline_data(ctrlc))
} }
PipelineData::Stream(stream, ..) => { PipelineData::ListStream(stream, ..) => {
Ok(stream.map(f).flatten().into_pipeline_data(ctrlc)) Ok(stream.map(f).flatten().into_pipeline_data(ctrlc))
} }
PipelineData::StringStream(stream, span, ..) => Ok(stream
.map(move |x| match x {
Ok(s) => Value::String { val: s, span },
Err(err) => Value::Error { error: err },
})
.map(f)
.flatten()
.into_pipeline_data(ctrlc)),
PipelineData::Value(Value::Range { val, .. }, ..) => match val.into_range_iter() { PipelineData::Value(Value::Range { val, .. }, ..) => match val.into_range_iter() {
Ok(iter) => Ok(iter.map(f).flatten().into_pipeline_data(ctrlc)), Ok(iter) => Ok(iter.map(f).flatten().into_pipeline_data(ctrlc)),
Err(error) => Err(error), Err(error) => Err(error),
}, },
PipelineData::Value(v, ..) => Ok(f(v).into_iter().into_pipeline_data(ctrlc)), PipelineData::Value(v, ..) => Ok(f(v).into_iter().into_pipeline_data(ctrlc)),
PipelineData::ByteStream(_, span, ..) => Err(ShellError::UnsupportedInput(
"Binary output from this command may need to be decoded using the 'decode' command"
.into(),
span,
)),
} }
} }
@ -185,7 +240,15 @@ impl PipelineData {
PipelineData::Value(Value::List { vals, .. }, ..) => { PipelineData::Value(Value::List { vals, .. }, ..) => {
Ok(vals.into_iter().filter(f).into_pipeline_data(ctrlc)) Ok(vals.into_iter().filter(f).into_pipeline_data(ctrlc))
} }
PipelineData::Stream(stream, ..) => Ok(stream.filter(f).into_pipeline_data(ctrlc)), PipelineData::ListStream(stream, ..) => Ok(stream.filter(f).into_pipeline_data(ctrlc)),
PipelineData::StringStream(stream, span, ..) => Ok(stream
.map(move |x| match x {
Ok(s) => Value::String { val: s, span },
Err(err) => Value::Error { error: err },
})
.filter(f)
.into_pipeline_data(ctrlc)),
PipelineData::Value(Value::Range { val, .. }, ..) => { PipelineData::Value(Value::Range { val, .. }, ..) => {
Ok(val.into_range_iter()?.filter(f).into_pipeline_data(ctrlc)) Ok(val.into_range_iter()?.filter(f).into_pipeline_data(ctrlc))
} }
@ -196,16 +259,15 @@ impl PipelineData {
Ok(Value::Nothing { span: v.span()? }.into_pipeline_data()) Ok(Value::Nothing { span: v.span()? }.into_pipeline_data())
} }
} }
PipelineData::ByteStream(_, span, ..) => Err(ShellError::UnsupportedInput(
"Binary output from this command may need to be decoded using the 'decode' command"
.into(),
span,
)),
} }
} }
} }
// impl Default for PipelineData {
// fn default() -> Self {
// PipelineData::new()
// }
// }
pub struct PipelineIterator(PipelineData); pub struct PipelineIterator(PipelineData);
impl IntoIterator for PipelineData { impl IntoIterator for PipelineData {
@ -216,7 +278,7 @@ impl IntoIterator for PipelineData {
fn into_iter(self) -> Self::IntoIter { fn into_iter(self) -> Self::IntoIter {
match self { match self {
PipelineData::Value(Value::List { vals, .. }, metadata) => { PipelineData::Value(Value::List { vals, .. }, metadata) => {
PipelineIterator(PipelineData::Stream( PipelineIterator(PipelineData::ListStream(
ValueStream { ValueStream {
stream: Box::new(vals.into_iter()), stream: Box::new(vals.into_iter()),
ctrlc: None, ctrlc: None,
@ -226,14 +288,14 @@ impl IntoIterator for PipelineData {
} }
PipelineData::Value(Value::Range { val, .. }, metadata) => { PipelineData::Value(Value::Range { val, .. }, metadata) => {
match val.into_range_iter() { match val.into_range_iter() {
Ok(iter) => PipelineIterator(PipelineData::Stream( Ok(iter) => PipelineIterator(PipelineData::ListStream(
ValueStream { ValueStream {
stream: Box::new(iter), stream: Box::new(iter),
ctrlc: None, ctrlc: None,
}, },
metadata, metadata,
)), )),
Err(error) => PipelineIterator(PipelineData::Stream( Err(error) => PipelineIterator(PipelineData::ListStream(
ValueStream { ValueStream {
stream: Box::new(std::iter::once(Value::Error { error })), stream: Box::new(std::iter::once(Value::Error { error })),
ctrlc: None, ctrlc: None,
@ -254,7 +316,18 @@ impl Iterator for PipelineIterator {
match &mut self.0 { match &mut self.0 {
PipelineData::Value(Value::Nothing { .. }, ..) => None, PipelineData::Value(Value::Nothing { .. }, ..) => None,
PipelineData::Value(v, ..) => Some(std::mem::take(v)), PipelineData::Value(v, ..) => Some(std::mem::take(v)),
PipelineData::Stream(stream, ..) => stream.next(), PipelineData::ListStream(stream, ..) => stream.next(),
PipelineData::StringStream(stream, span, ..) => stream.next().map(|x| match x {
Ok(x) => Value::String {
val: x,
span: *span,
},
Err(err) => Value::Error { error: err },
}),
PipelineData::ByteStream(stream, span, ..) => stream.next().map(|x| Value::Binary {
val: x,
span: *span,
}),
} }
} }
} }
@ -288,7 +361,7 @@ where
<I::IntoIter as Iterator>::Item: Into<Value>, <I::IntoIter as Iterator>::Item: Into<Value>,
{ {
fn into_pipeline_data(self, ctrlc: Option<Arc<AtomicBool>>) -> PipelineData { fn into_pipeline_data(self, ctrlc: Option<Arc<AtomicBool>>) -> PipelineData {
PipelineData::Stream( PipelineData::ListStream(
ValueStream { ValueStream {
stream: Box::new(self.into_iter().map(Into::into)), stream: Box::new(self.into_iter().map(Into::into)),
ctrlc, ctrlc,
@ -302,7 +375,7 @@ where
metadata: PipelineMetadata, metadata: PipelineMetadata,
ctrlc: Option<Arc<AtomicBool>>, ctrlc: Option<Arc<AtomicBool>>,
) -> PipelineData { ) -> PipelineData {
PipelineData::Stream( PipelineData::ListStream(
ValueStream { ValueStream {
stream: Box::new(self.into_iter().map(Into::into)), stream: Box::new(self.into_iter().map(Into::into)),
ctrlc, ctrlc,

88
crates/nu-protocol/src/value/stream.rs
View file

@ -7,6 +7,94 @@ use std::{
}, },
}; };
/// A single buffer of binary data streamed over multiple parts. Optionally contains ctrl-c that can be used
/// to break the stream.
pub struct ByteStream {
pub stream: Box<dyn Iterator<Item = Vec<u8>> + Send + 'static>,
pub ctrlc: Option<Arc<AtomicBool>>,
}
impl ByteStream {
pub fn into_vec(self) -> Vec<u8> {
self.flatten().collect::<Vec<u8>>()
}
}
impl Debug for ByteStream {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ByteStream").finish()
}
}
impl Iterator for ByteStream {
type Item = Vec<u8>;
fn next(&mut self) -> Option<Self::Item> {
if let Some(ctrlc) = &self.ctrlc {
if ctrlc.load(Ordering::SeqCst) {
None
} else {
self.stream.next()
}
} else {
self.stream.next()
}
}
}
/// A single string streamed over multiple parts. Optionally contains ctrl-c that can be used
/// to break the stream.
pub struct StringStream {
pub stream: Box<dyn Iterator<Item = Result<String, ShellError>> + Send + 'static>,
pub ctrlc: Option<Arc<AtomicBool>>,
}
impl StringStream {
pub fn into_string(self, separator: &str) -> Result<String, ShellError> {
let mut output = String::new();
let mut first = true;
for s in self.stream {
output.push_str(&s?);
if !first {
output.push_str(separator);
} else {
first = false;
}
}
Ok(output)
}
pub fn from_stream(
input: impl Iterator<Item = Result<String, ShellError>> + Send + 'static,
ctrlc: Option<Arc<AtomicBool>>,
) -> StringStream {
StringStream {
stream: Box::new(input),
ctrlc,
}
}
}
impl Debug for StringStream {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("StringStream").finish()
}
}
impl Iterator for StringStream {
type Item = Result<String, ShellError>;
fn next(&mut self) -> Option<Self::Item> {
if let Some(ctrlc) = &self.ctrlc {
if ctrlc.load(Ordering::SeqCst) {
None
} else {
self.stream.next()
}
} else {
self.stream.next()
}
}
}
/// A potentially infinite stream of values, optinally with a mean to send a Ctrl-C signal to stop /// A potentially infinite stream of values, optinally with a mean to send a Ctrl-C signal to stop
/// the stream from continuing. /// the stream from continuing.
/// ///

27
src/main.rs
View file

@ -583,6 +583,28 @@ fn print_pipeline_data(
let config = stack.get_config().unwrap_or_default(); let config = stack.get_config().unwrap_or_default();
match input {
PipelineData::StringStream(stream, _, _) => {
for s in stream {
print!("{}", s?);
let _ = std::io::stdout().flush();
}
return Ok(());
}
PipelineData::ByteStream(stream, _, _) => {
for v in stream {
let s = if v.iter().all(|x| x.is_ascii()) {
format!("{}", String::from_utf8_lossy(&v))
} else {
format!("{}\n", nu_pretty_hex::pretty_hex(&v))
};
println!("{}", s);
}
return Ok(());
}
_ => {}
}
match engine_state.find_decl("table".as_bytes()) { match engine_state.find_decl("table".as_bytes()) {
Some(decl_id) => { Some(decl_id) => {
let table = let table =
@ -663,7 +685,12 @@ fn update_prompt<'prompt>(
} }
}; };
match evaluated_prompt {
Ok(evaluated_prompt) => {
nu_prompt.update_prompt(evaluated_prompt); nu_prompt.update_prompt(evaluated_prompt);
}
_ => nu_prompt.update_prompt(String::new()),
}
nu_prompt as &dyn Prompt nu_prompt as &dyn Prompt
} }