From f05162811cac13a08f5edda7e69d90b2fc423c66 Mon Sep 17 00:00:00 2001 From: Simon Curtis <43214378+simon-curtis@users.noreply.github.com> Date: Sat, 11 Jan 2025 21:28:08 +0000 Subject: [PATCH] Implementing ByteStream interuption on infinite stream (#13552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description This PR should address #13530 by explicitly handling ByteStreams. The issue can be replicated easily on linux by running: ```nushell open /dev/urandom | into binary | bytes at ..10 ``` Would leave the output hanging and with no way to cancel it, this was likely because it was trying to collect the input stream and would not complete. I have also put in an error to say that using negative offsets for a bytestream without a length cannot be used. ```nushell ~/git/nushell> open /dev/urandom | into binary | bytes at (-1).. Error: nu::shell::incorrect_value × Incorrect value. ╭─[entry #3:1:35] 1 │ open /dev/urandom | into binary | bytes at (-1).. · ────┬─── ───┬── · │ ╰── encountered here · ╰── Negative range values cannot be used with streams that don't specify a length ╰──── ``` # User-Facing Changes No operation changes, only the warning you get back for negative offsets # Tests + Formatting Ran `toolkit check pr ` with no errors or warnings Manual testing of the example commands above --------- Co-authored-by: Ian Manske Co-authored-by: Simon Curtis --- crates/nu-cmd-extra/src/example_test.rs | 7 +- crates/nu-cmd-lang/src/example_support.rs | 15 +- crates/nu-cmd-lang/src/example_test.rs | 7 +- crates/nu-command/src/bytes/at.rs | 155 ++++++---- crates/nu-command/src/example_test.rs | 7 +- crates/nu-command/src/filters/skip/skip_.rs | 22 +- crates/nu-command/src/filters/take/take_.rs | 20 +- crates/nu-command/tests/commands/bytes/at.rs | 72 +++++ crates/nu-command/tests/commands/bytes/mod.rs | 1 + crates/nu-protocol/src/errors/shell_error.rs | 12 + .../nu-protocol/src/pipeline/byte_stream.rs | 76 ++++- crates/nu-protocol/src/value/range.rs | 51 ++++ crates/nu-protocol/tests/mod.rs | 1 + .../nu-protocol/tests/pipeline/byte_stream.rs | 268 ++++++++++++++++++ crates/nu-protocol/tests/pipeline/mod.rs | 1 + 15 files changed, 612 insertions(+), 103 deletions(-) create mode 100644 crates/nu-command/tests/commands/bytes/at.rs create mode 100644 crates/nu-protocol/tests/mod.rs create mode 100644 crates/nu-protocol/tests/pipeline/byte_stream.rs create mode 100644 crates/nu-protocol/tests/pipeline/mod.rs diff --git a/crates/nu-cmd-extra/src/example_test.rs b/crates/nu-cmd-extra/src/example_test.rs index 082080f8f9..9ce94bf05b 100644 --- a/crates/nu-cmd-extra/src/example_test.rs +++ b/crates/nu-cmd-extra/src/example_test.rs @@ -43,7 +43,12 @@ mod test_examples { signature.operates_on_cell_paths(), ), ); - check_example_evaluates_to_expected_output(&example, cwd.as_path(), &mut engine_state); + check_example_evaluates_to_expected_output( + cmd.name(), + &example, + cwd.as_path(), + &mut engine_state, + ); } check_all_signature_input_output_types_entries_have_examples( diff --git a/crates/nu-cmd-lang/src/example_support.rs b/crates/nu-cmd-lang/src/example_support.rs index a26876b78c..0ff4428d1a 100644 --- a/crates/nu-cmd-lang/src/example_support.rs +++ b/crates/nu-cmd-lang/src/example_support.rs @@ -139,6 +139,7 @@ pub fn eval_block( } pub fn check_example_evaluates_to_expected_output( + cmd_name: &str, example: &Example, cwd: &std::path::Path, engine_state: &mut Box, @@ -159,11 +160,17 @@ pub fn check_example_evaluates_to_expected_output( // If the command you are testing requires to compare another case, then // you need to define its equality in the Value struct if let Some(expected) = example.result.as_ref() { + let expected = DebuggableValue(expected); + let result = DebuggableValue(&result); assert_eq!( - DebuggableValue(&result), - DebuggableValue(expected), - "The example result differs from the expected value", - ) + result, + expected, + "Error: The result of example '{}' for the command '{}' differs from the expected value.\n\nExpected: {:?}\nActual: {:?}\n", + example.description, + cmd_name, + expected, + result, + ); } } diff --git a/crates/nu-cmd-lang/src/example_test.rs b/crates/nu-cmd-lang/src/example_test.rs index 09c01234f3..e66c8bf5f2 100644 --- a/crates/nu-cmd-lang/src/example_test.rs +++ b/crates/nu-cmd-lang/src/example_test.rs @@ -44,7 +44,12 @@ mod test_examples { signature.operates_on_cell_paths(), ), ); - check_example_evaluates_to_expected_output(&example, cwd.as_path(), &mut engine_state); + check_example_evaluates_to_expected_output( + cmd.name(), + &example, + cwd.as_path(), + &mut engine_state, + ); } check_all_signature_input_output_types_entries_have_examples( diff --git a/crates/nu-command/src/bytes/at.rs b/crates/nu-command/src/bytes/at.rs index adb4bfd9c2..9663e545a4 100644 --- a/crates/nu-command/src/bytes/at.rs +++ b/crates/nu-command/src/bytes/at.rs @@ -1,15 +1,14 @@ -use nu_cmd_base::{ - input_handler::{operate, CmdArgument}, - util, -}; +use std::ops::Bound; + +use nu_cmd_base::input_handler::{operate, CmdArgument}; use nu_engine::command_prelude::*; -use nu_protocol::Range; +use nu_protocol::{IntRange, Range}; #[derive(Clone)] pub struct BytesAt; struct Arguments { - indexes: Subbytes, + range: IntRange, cell_paths: Option>, } @@ -19,15 +18,6 @@ impl CmdArgument for Arguments { } } -impl From<(isize, isize)> for Subbytes { - fn from(input: (isize, isize)) -> Self { - Self(input.0, input.1) - } -} - -#[derive(Clone, Copy)] -struct Subbytes(isize, isize); - impl Command for BytesAt { fn name(&self) -> &str { "bytes at" @@ -44,6 +34,7 @@ impl Command for BytesAt { (Type::table(), Type::table()), (Type::record(), Type::record()), ]) + .allow_variants_without_examples(true) .required("range", SyntaxShape::Range, "The range to get bytes.") .rest( "rest", @@ -68,86 +59,115 @@ impl Command for BytesAt { call: &Call, input: PipelineData, ) -> Result { - let range: Range = call.req(engine_state, stack, 0)?; - let indexes = match util::process_range(&range) { - Ok(idxs) => idxs.into(), - Err(processing_error) => { - return Err(processing_error("could not perform subbytes", call.head)); + let range = match call.req(engine_state, stack, 0)? { + Range::IntRange(range) => range, + _ => { + return Err(ShellError::UnsupportedInput { + msg: "Float ranges are not supported for byte streams".into(), + input: "value originates from here".into(), + msg_span: call.head, + input_span: call.head, + }) } }; let cell_paths: Vec = call.rest(engine_state, stack, 1)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - let args = Arguments { - indexes, - cell_paths, - }; - operate(action, args, input, call.head, engine_state.signals()) + if let PipelineData::ByteStream(stream, metadata) = input { + let stream = stream.slice(call.head, call.arguments_span(), range)?; + Ok(PipelineData::ByteStream(stream, metadata)) + } else { + operate( + map_value, + Arguments { range, cell_paths }, + input, + call.head, + engine_state.signals(), + ) + } } fn examples(&self) -> Vec { vec![ Example { - description: "Get a subbytes `0x[10 01]` from the bytes `0x[33 44 55 10 01 13]`", - example: " 0x[33 44 55 10 01 13] | bytes at 3..<4", - result: Some(Value::test_binary(vec![0x10])), - }, - Example { - description: "Get a subbytes `0x[10 01 13]` from the bytes `0x[33 44 55 10 01 13]`", - example: " 0x[33 44 55 10 01 13] | bytes at 3..6", - result: Some(Value::test_binary(vec![0x10, 0x01, 0x13])), - }, - Example { - description: "Get the remaining characters from a starting index", - example: " { data: 0x[33 44 55 10 01 13] } | bytes at 3.. data", + description: "Extract bytes starting from a specific index", + example: "{ data: 0x[33 44 55 10 01 13 10] } | bytes at 3.. data", result: Some(Value::test_record(record! { - "data" => Value::test_binary(vec![0x10, 0x01, 0x13]), + "data" => Value::test_binary(vec![0x10, 0x01, 0x13, 0x10]), })), }, Example { - description: "Get the characters from the beginning until ending index", - example: " 0x[33 44 55 10 01 13] | bytes at ..<4", + description: "Slice out `0x[10 01 13]` from `0x[33 44 55 10 01 13]`", + example: "0x[33 44 55 10 01 13] | bytes at 3..5", + result: Some(Value::test_binary(vec![0x10, 0x01, 0x13])), + }, + Example { + description: "Extract bytes from the start up to a specific index", + example: "0x[33 44 55 10 01 13 10] | bytes at ..4", + result: Some(Value::test_binary(vec![0x33, 0x44, 0x55, 0x10, 0x01])), + }, + Example { + description: "Extract byte `0x[10]` using an exclusive end index", + example: "0x[33 44 55 10 01 13 10] | bytes at 3..<4", + result: Some(Value::test_binary(vec![0x10])), + }, + Example { + description: "Extract bytes up to a negative index (inclusive)", + example: "0x[33 44 55 10 01 13 10] | bytes at ..-4", result: Some(Value::test_binary(vec![0x33, 0x44, 0x55, 0x10])), }, Example { - description: - "Or the characters from the beginning until ending index inside a table", - example: r#" [[ColA ColB ColC]; [0x[11 12 13] 0x[14 15 16] 0x[17 18 19]]] | bytes at 1.. ColB ColC"#, + description: "Slice bytes across multiple table columns", + example: r#"[[ColA ColB ColC]; [0x[11 12 13] 0x[14 15 16] 0x[17 18 19]]] | bytes at 1.. ColB ColC"#, result: Some(Value::test_list(vec![Value::test_record(record! { "ColA" => Value::test_binary(vec![0x11, 0x12, 0x13]), "ColB" => Value::test_binary(vec![0x15, 0x16]), "ColC" => Value::test_binary(vec![0x18, 0x19]), })])), }, + Example { + description: "Extract the last three bytes using a negative start index", + example: "0x[33 44 55 10 01 13 10] | bytes at (-3)..", + result: Some(Value::test_binary(vec![0x01, 0x13, 0x10])), + }, ] } } -fn action(input: &Value, args: &Arguments, head: Span) -> Value { - let range = &args.indexes; +fn map_value(input: &Value, args: &Arguments, head: Span) -> Value { + let range = &args.range; match input { Value::Binary { val, .. } => { - let len = val.len() as isize; - let start = if range.0 < 0 { range.0 + len } else { range.0 }; - let end = if range.1 < 0 { range.1 + len } else { range.1 }; + let len = val.len() as u64; + let start: u64 = range.absolute_start(len); + let start: usize = match start.try_into() { + Ok(start) => start, + Err(_) => { + let span = input.span(); + return Value::error( + ShellError::UnsupportedInput { + msg: format!( + "Absolute start position {start} was too large for your system arch." + ), + input: args.range.to_string(), + msg_span: span, + input_span: span, + }, + head, + ); + } + }; - if start > end { - Value::binary(vec![], head) - } else { - let val_iter = val.iter().skip(start as usize); - Value::binary( - if end == isize::MAX { - val_iter.copied().collect::>() - } else { - val_iter.take((end - start + 1) as usize).copied().collect() - }, - head, - ) - } + let bytes: Vec = match range.absolute_end(len) { + Bound::Unbounded => val[start..].into(), + Bound::Included(end) => val[start..=end as usize].into(), + Bound::Excluded(end) => val[start..end as usize].into(), + }; + + Value::binary(bytes, head) } Value::Error { .. } => input.clone(), - other => Value::error( ShellError::UnsupportedInput { msg: "Only binary values are supported".into(), @@ -159,3 +179,14 @@ fn action(input: &Value, args: &Arguments, head: Span) -> Value { ), } } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_examples() { + use crate::test_examples; + test_examples(BytesAt {}) + } +} diff --git a/crates/nu-command/src/example_test.rs b/crates/nu-command/src/example_test.rs index 4879cab0a9..dca285f835 100644 --- a/crates/nu-command/src/example_test.rs +++ b/crates/nu-command/src/example_test.rs @@ -61,7 +61,12 @@ mod test_examples { signature.operates_on_cell_paths(), ), ); - check_example_evaluates_to_expected_output(&example, cwd.as_path(), &mut engine_state); + check_example_evaluates_to_expected_output( + cmd.name(), + &example, + cwd.as_path(), + &mut engine_state, + ); } check_all_signature_input_output_types_entries_have_examples( diff --git a/crates/nu-command/src/filters/skip/skip_.rs b/crates/nu-command/src/filters/skip/skip_.rs index fade5a00ba..d4adca1455 100644 --- a/crates/nu-command/src/filters/skip/skip_.rs +++ b/crates/nu-command/src/filters/skip/skip_.rs @@ -1,6 +1,4 @@ use nu_engine::command_prelude::*; -use nu_protocol::Signals; -use std::io::{self, Read}; #[derive(Clone)] pub struct Skip; @@ -96,22 +94,10 @@ impl Command for Skip { PipelineData::ByteStream(stream, metadata) => { if stream.type_().is_binary_coercible() { let span = stream.span(); - if let Some(mut reader) = stream.reader() { - // Copy the number of skipped bytes into the sink before proceeding - io::copy(&mut (&mut reader).take(n as u64), &mut io::sink()) - .err_span(span)?; - Ok(PipelineData::ByteStream( - ByteStream::read( - reader, - call.head, - Signals::empty(), - ByteStreamType::Binary, - ), - metadata, - )) - } else { - Ok(PipelineData::Empty) - } + Ok(PipelineData::ByteStream( + stream.skip(span, n as u64)?, + metadata, + )) } else { Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), diff --git a/crates/nu-command/src/filters/take/take_.rs b/crates/nu-command/src/filters/take/take_.rs index 85a94fcd4b..a03b1825da 100644 --- a/crates/nu-command/src/filters/take/take_.rs +++ b/crates/nu-command/src/filters/take/take_.rs @@ -1,6 +1,5 @@ use nu_engine::command_prelude::*; use nu_protocol::Signals; -use std::io::Read; #[derive(Clone)] pub struct Take; @@ -89,20 +88,11 @@ impl Command for Take { )), PipelineData::ByteStream(stream, metadata) => { if stream.type_().is_binary_coercible() { - if let Some(reader) = stream.reader() { - // Just take 'rows' bytes off the stream, mimicking the binary behavior - Ok(PipelineData::ByteStream( - ByteStream::read( - reader.take(rows_desired as u64), - head, - Signals::empty(), - ByteStreamType::Binary, - ), - metadata, - )) - } else { - Ok(PipelineData::Empty) - } + let span = stream.span(); + Ok(PipelineData::ByteStream( + stream.take(span, rows_desired as u64)?, + metadata, + )) } else { Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), diff --git a/crates/nu-command/tests/commands/bytes/at.rs b/crates/nu-command/tests/commands/bytes/at.rs new file mode 100644 index 0000000000..4a8d034b3f --- /dev/null +++ b/crates/nu-command/tests/commands/bytes/at.rs @@ -0,0 +1,72 @@ +use nu_test_support::nu; + +#[test] +pub fn returns_error_for_relative_range_on_infinite_stream() { + let actual = nu!("nu --testbin iecho 3 | bytes at ..-3"); + assert!( + actual.err.contains( + "Relative range values cannot be used with streams that don't have a known length" + ), + "Expected error message for negative range with infinite stream" + ); +} + +#[test] +pub fn returns_bytes_for_fixed_range_on_infinite_stream_including_end() { + let actual = nu!("nu --testbin iecho 3 | bytes at ..10 | decode"); + assert_eq!( + actual.out, "33333", + "Expected bytes from index 0 to 10, but got different output" + ); +} + +#[test] +pub fn returns_bytes_for_fixed_range_on_infinite_stream_excluding_end() { + let actual = nu!("nu --testbin iecho 3 | bytes at ..<9 | decode"); + assert_eq!( + actual.out, "3333", + "Expected bytes from index 0 to 8, but got different output" + ); +} + +#[test] +pub fn test_string_returns_correct_slice_for_simple_positive_slice() { + let actual = nu!("\"Hello World\" | encode utf8 | bytes at ..4 | decode"); + assert_eq!(actual.out, "Hello"); +} + +#[test] +pub fn test_string_returns_correct_slice_for_negative_start() { + let actual = nu!("\"Hello World\" | encode utf8 | bytes at (-5)..10 | decode"); + assert_eq!(actual.out, "World"); +} + +#[test] +pub fn test_string_returns_correct_slice_for_negative_end() { + let actual = nu!("\"Hello World\" | encode utf8 | bytes at ..-7 | decode"); + assert_eq!(actual.out, "Hello"); +} + +#[test] +pub fn test_string_returns_correct_slice_for_empty_slice() { + let actual = nu!("\"Hello World\" | encode utf8 | bytes at 5..<5 | decode"); + assert_eq!(actual.out, ""); +} + +#[test] +pub fn test_string_returns_correct_slice_for_out_of_bounds() { + let actual = nu!("\"Hello World\" | encode utf8 | bytes at ..20 | decode"); + assert_eq!(actual.out, "Hello World"); +} + +#[test] +pub fn test_string_returns_correct_slice_for_invalid_range() { + let actual = nu!("\"Hello World\" | encode utf8 | bytes at 11..5 | decode"); + assert_eq!(actual.out, ""); +} + +#[test] +pub fn test_string_returns_correct_slice_for_max_end() { + let actual = nu!("\"Hello World\" | encode utf8 | bytes at 6..<11 | decode"); + assert_eq!(actual.out, "World"); +} diff --git a/crates/nu-command/tests/commands/bytes/mod.rs b/crates/nu-command/tests/commands/bytes/mod.rs index 10b2a494f8..f1faaef005 100644 --- a/crates/nu-command/tests/commands/bytes/mod.rs +++ b/crates/nu-command/tests/commands/bytes/mod.rs @@ -1 +1,2 @@ +mod at; mod collect; diff --git a/crates/nu-protocol/src/errors/shell_error.rs b/crates/nu-protocol/src/errors/shell_error.rs index 61bd6ce11e..208f2178f6 100644 --- a/crates/nu-protocol/src/errors/shell_error.rs +++ b/crates/nu-protocol/src/errors/shell_error.rs @@ -653,6 +653,18 @@ pub enum ShellError { creation_site: Span, }, + /// Attempted to us a relative range on an infinite stream + /// + /// ## Resolution + /// + /// Ensure that either the range is absolute or the stream has a known length. + #[error("Relative range values cannot be used with streams that don't have a known length")] + #[diagnostic(code(nu::shell::relative_range_on_infinite_stream))] + RelativeRangeOnInfiniteStream { + #[label = "Relative range values cannot be used with streams that don't have a known length"] + span: Span, + }, + /// An error happened while performing an external command. /// /// ## Resolution diff --git a/crates/nu-protocol/src/pipeline/byte_stream.rs b/crates/nu-protocol/src/pipeline/byte_stream.rs index e24c0753e6..b5cbbff237 100644 --- a/crates/nu-protocol/src/pipeline/byte_stream.rs +++ b/crates/nu-protocol/src/pipeline/byte_stream.rs @@ -1,8 +1,9 @@ //! Module managing the streaming of raw bytes between pipeline elements #[cfg(feature = "os")] use crate::process::{ChildPipe, ChildProcess}; -use crate::{ErrSpan, IntoSpanned, PipelineData, ShellError, Signals, Span, Type, Value}; +use crate::{ErrSpan, IntRange, IntoSpanned, PipelineData, ShellError, Signals, Span, Type, Value}; use serde::{Deserialize, Serialize}; +use std::ops::Bound; #[cfg(unix)] use std::os::fd::OwnedFd; #[cfg(windows)] @@ -220,6 +221,79 @@ impl ByteStream { ) } + pub fn skip(self, span: Span, n: u64) -> Result { + let known_size = self.known_size.map(|len| len.saturating_sub(n)); + if let Some(mut reader) = self.reader() { + // Copy the number of skipped bytes into the sink before proceeding + io::copy(&mut (&mut reader).take(n), &mut io::sink()).err_span(span)?; + Ok( + ByteStream::read(reader, span, Signals::empty(), ByteStreamType::Binary) + .with_known_size(known_size), + ) + } else { + Err(ShellError::TypeMismatch { + err_message: "expected readable stream".into(), + span, + }) + } + } + + pub fn take(self, span: Span, n: u64) -> Result { + let known_size = self.known_size.map(|s| s.min(n)); + if let Some(reader) = self.reader() { + Ok(ByteStream::read( + reader.take(n), + span, + Signals::empty(), + ByteStreamType::Binary, + ) + .with_known_size(known_size)) + } else { + Err(ShellError::TypeMismatch { + err_message: "expected readable stream".into(), + span, + }) + } + } + + pub fn slice( + self, + val_span: Span, + call_span: Span, + range: IntRange, + ) -> Result { + if let Some(len) = self.known_size { + let start = range.absolute_start(len); + let stream = self.skip(val_span, start); + + match range.absolute_end(len) { + Bound::Unbounded => stream, + Bound::Included(end) | Bound::Excluded(end) if end < start => { + stream.and_then(|s| s.take(val_span, 0)) + } + Bound::Included(end) => { + let distance = end - start + 1; + stream.and_then(|s| s.take(val_span, distance.min(len))) + } + Bound::Excluded(end) => { + let distance = end - start; + stream.and_then(|s| s.take(val_span, distance.min(len))) + } + } + } else if range.is_relative() { + Err(ShellError::RelativeRangeOnInfiniteStream { span: call_span }) + } else { + let start = range.start() as u64; + let stream = self.skip(val_span, start); + + match range.distance() { + Bound::Unbounded => stream, + Bound::Included(distance) => stream.and_then(|s| s.take(val_span, distance)), + Bound::Excluded(distance) => stream.and_then(|s| s.take(val_span, distance - 1)), + } + } + } + /// Create a [`ByteStream`] from a string. The type of the stream is always `String`. pub fn read_string(string: String, span: Span, signals: Signals) -> Self { let len = string.len(); diff --git a/crates/nu-protocol/src/value/range.rs b/crates/nu-protocol/src/value/range.rs index c4d6d4655e..268b24b623 100644 --- a/crates/nu-protocol/src/value/range.rs +++ b/crates/nu-protocol/src/value/range.rs @@ -81,10 +81,46 @@ mod int_range { self.start } + // Resolves the absolute start position given the length of the input value + pub fn absolute_start(&self, len: u64) -> u64 { + let max_index = len - 1; + match self.start { + start if start < 0 => len.saturating_sub(start.unsigned_abs()), + start => max_index.min(start as u64), + } + } + + /// Returns the distance between the start and end of the range + /// The result will always be 0 or positive + pub fn distance(&self) -> Bound { + match self.end { + Bound::Unbounded => Bound::Unbounded, + Bound::Included(end) if self.start > end => Bound::Included(0), + Bound::Excluded(end) if self.start > end => Bound::Excluded(0), + Bound::Included(end) => Bound::Included((end - self.start) as u64), + Bound::Excluded(end) => Bound::Excluded((end - self.start) as u64), + } + } + pub fn end(&self) -> Bound { self.end } + pub fn absolute_end(&self, len: u64) -> Bound { + let max_index = len - 1; + match self.end { + Bound::Unbounded => Bound::Unbounded, + Bound::Included(i) => Bound::Included(match i { + i if i < 0 => len.saturating_sub(i.unsigned_abs()), + i => max_index.min(i as u64), + }), + Bound::Excluded(i) => Bound::Excluded(match i { + i if i < 0 => len.saturating_sub(i.unsigned_abs()), + i => len.min(i as u64), + }), + } + } + pub fn step(&self) -> i64 { self.step } @@ -93,6 +129,21 @@ mod int_range { self.end == Bound::Unbounded } + pub fn is_relative(&self) -> bool { + self.is_start_relative() || self.is_end_relative() + } + + pub fn is_start_relative(&self) -> bool { + self.start < 0 + } + + pub fn is_end_relative(&self) -> bool { + match self.end { + Bound::Included(end) | Bound::Excluded(end) => end < 0, + _ => false, + } + } + pub fn contains(&self, value: i64) -> bool { if self.step < 0 { // Decreasing range diff --git a/crates/nu-protocol/tests/mod.rs b/crates/nu-protocol/tests/mod.rs new file mode 100644 index 0000000000..eab2e1f414 --- /dev/null +++ b/crates/nu-protocol/tests/mod.rs @@ -0,0 +1 @@ +mod pipeline; diff --git a/crates/nu-protocol/tests/pipeline/byte_stream.rs b/crates/nu-protocol/tests/pipeline/byte_stream.rs new file mode 100644 index 0000000000..3d06573461 --- /dev/null +++ b/crates/nu-protocol/tests/pipeline/byte_stream.rs @@ -0,0 +1,268 @@ +use nu_protocol::{ast::RangeInclusion, ByteStream, IntRange, Signals, Span, Value}; + +#[test] +pub fn test_simple_positive_slice_exclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(0, 5, RangeInclusion::RightExclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, "Hello"); +} + +#[test] +pub fn test_negative_start_exclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(-5, 11, RangeInclusion::RightExclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, "World"); +} + +#[test] +pub fn test_negative_end_exclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(0, -6, RangeInclusion::RightExclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, "Hello"); +} + +#[test] +pub fn test_negative_start_and_end_exclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(-5, -2, RangeInclusion::RightExclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, "Wor"); +} + +#[test] +pub fn test_empty_slice_exclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(5, 5, RangeInclusion::RightExclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, ""); +} + +#[test] +pub fn test_out_of_bounds_exclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(0, 20, RangeInclusion::RightExclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, "Hello World"); +} + +#[test] +pub fn test_invalid_range_exclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(11, 5, RangeInclusion::RightExclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, ""); +} + +#[test] +pub fn test_max_end_exclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(6, i64::MAX, RangeInclusion::RightExclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, "World"); +} + +#[test] +pub fn test_simple_positive_slice_inclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(0, 5, RangeInclusion::RightExclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, "Hello"); +} + +#[test] +pub fn test_negative_start_inclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(-5, 11, RangeInclusion::Inclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, "World"); +} + +#[test] +pub fn test_negative_end_inclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(0, -7, RangeInclusion::Inclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, "Hello"); +} + +#[test] +pub fn test_negative_start_and_end_inclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(-5, -1, RangeInclusion::Inclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, "World"); +} + +#[test] +pub fn test_empty_slice_inclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(5, 5, RangeInclusion::Inclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, " "); +} + +#[test] +pub fn test_out_of_bounds_inclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(0, 20, RangeInclusion::Inclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, "Hello World"); +} + +#[test] +pub fn test_invalid_range_inclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(11, 5, RangeInclusion::Inclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, ""); +} + +#[test] +pub fn test_max_end_inclusive() { + let data = b"Hello World".to_vec(); + let stream = ByteStream::read_binary(data, Span::test_data(), Signals::empty()); + let sliced = stream + .slice( + Span::test_data(), + Span::test_data(), + create_range(6, i64::MAX, RangeInclusion::Inclusive), + ) + .unwrap(); + let result = sliced.into_bytes().unwrap(); + let result = String::from_utf8(result).unwrap(); + assert_eq!(result, "World"); +} + +fn create_range(start: i64, end: i64, inclusion: RangeInclusion) -> IntRange { + IntRange::new( + Value::int(start, Span::unknown()), + Value::nothing(Span::test_data()), + Value::int(end, Span::unknown()), + inclusion, + Span::unknown(), + ) + .unwrap() +} diff --git a/crates/nu-protocol/tests/pipeline/mod.rs b/crates/nu-protocol/tests/pipeline/mod.rs new file mode 100644 index 0000000000..a3003e8031 --- /dev/null +++ b/crates/nu-protocol/tests/pipeline/mod.rs @@ -0,0 +1 @@ +mod byte_stream;