nushell/crates/nu-parser/tests/test_parser.rs

2442 lines
76 KiB
Rust
Raw Normal View History

2021-08-30 18:36:07 +00:00
use nu_parser::*;
2021-09-02 18:21:37 +00:00
use nu_protocol::{
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
ast::{Argument, Call, Expr, Expression, ExternalArgument, PathMember, Range},
2021-10-25 06:31:39 +00:00
engine::{Command, EngineState, Stack, StateWorkingSet},
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
ParseError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
2021-09-02 18:21:37 +00:00
};
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
use rstest::rstest;
2021-08-30 18:36:07 +00:00
2021-09-13 08:19:05 +00:00
#[cfg(test)]
2021-10-25 04:01:02 +00:00
#[derive(Clone)]
2021-09-13 08:19:05 +00:00
pub struct Let;
#[cfg(test)]
impl Command for Let {
fn name(&self) -> &str {
"let"
}
fn usage(&self) -> &str {
"Create a variable and give it a value."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build("let")
.required("var_name", SyntaxShape::VarWithOptType, "variable name")
.required(
"initial_value",
SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::MathExpression)),
2021-09-13 08:19:05 +00:00
"equals sign followed by value",
)
}
fn run(
&self,
2021-10-25 06:31:39 +00:00
_engine_state: &EngineState,
_stack: &mut Stack,
_call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
2021-09-13 08:19:05 +00:00
todo!()
}
}
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 16:09:50 +00:00
fn test_int(
test_tag: &str, // name of sub-test
test: &[u8], // input expression
expected_val: Expr, // (usually Expr::{Int,String, Float}, not ::BinOp...
expected_err: Option<&str>,
) // substring in error text
{
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, test, true);
let err = working_set.parse_errors.first();
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 16:09:50 +00:00
if let Some(err_pat) = expected_err {
if let Some(parse_err) = err {
let act_err = format!("{:?}", parse_err);
assert!(
act_err.contains(err_pat),
"{test_tag}: expected err to contain {err_pat}, but actual error was {act_err}"
);
} else {
assert!(
err.is_some(),
"{test_tag}: expected err containing {err_pat}, but no error returned"
);
}
} else {
assert!(err.is_none(), "{test_tag}: unexpected error {err:#?}");
assert_eq!(block.len(), 1, "{test_tag}: result block length > 1");
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 16:09:50 +00:00
assert_eq!(
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
pipeline.len(),
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 16:09:50 +00:00
1,
"{test_tag}: got multiple result expressions, expected 1"
);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
compare_rhs_binary_op(test_tag, &expected_val, &element.expr.expr);
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 16:09:50 +00:00
}
}
fn compare_rhs_binary_op(
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 16:09:50 +00:00
test_tag: &str,
expected: &Expr, // the rhs expr we hope to see (::Int, ::Float, not ::B)
observed: &Expr, // the Expr actually provided: can be ::Int, ::Float, ::String,
// or ::BinOp (in which case rhs is checked), or ::Call (in which case cmd is checked)
) {
match observed {
Expr::Int(..) | Expr::Float(..) | Expr::String(..) => {
assert_eq!(
expected, observed,
"{test_tag}: Expected: {expected:#?}, observed {observed:#?}"
);
}
Expr::BinaryOp(_, _, e) => {
let observed_expr = &e.expr;
// can't pattern match Box<Foo>, but can match the box, then deref in separate statement.
assert_eq!(
expected, observed_expr,
"{test_tag}: Expected: {expected:#?}, observed: {observed:#?}"
)
}
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
Expr::ExternalCall(e, _) => {
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 16:09:50 +00:00
let observed_expr = &e.expr;
assert_eq!(
expected, observed_expr,
"{test_tag}: Expected: {expected:#?}, observed: {observed_expr:#?}"
)
}
_ => {
panic!("{test_tag}: Unexpected Expr:: variant returned, observed {observed:#?}");
}
}
}
#[test]
pub fn multi_test_parse_int() {
struct Test<'a>(&'a str, &'a [u8], Expr, Option<&'a str>);
// use test expression of form '0 + x' to force parse() to parse x as numeric.
// if expression were just 'x', parse() would try other items that would mask the error we're looking for.
let tests = vec![
Test("binary literal int", b"0 + 0b0", Expr::Int(0), None),
Test(
"binary literal invalid digits",
b"0 + 0b2",
Expr::Int(0),
Some("invalid digits for radix 2"),
),
Test("octal literal int", b"0 + 0o1", Expr::Int(1), None),
Test(
"octal literal int invalid digits",
b"0 + 0o8",
Expr::Int(0),
Some("invalid digits for radix 8"),
),
Test(
"octal literal int truncated",
b"0 + 0o",
Expr::Int(0),
Some("invalid digits for radix 8"),
),
Test("hex literal int", b"0 + 0x2", Expr::Int(2), None),
Test(
"hex literal int invalid digits",
b"0 + 0x0aq",
Expr::Int(0),
Some("invalid digits for radix 16"),
),
Test(
"hex literal with 'e' not mistaken for float",
b"0 + 0x00e0",
Expr::Int(0xe0),
None,
),
// decimal (rad10) literal is anything that starts with
// optional sign then a digit.
Test("rad10 literal int", b"0 + 42", Expr::Int(42), None),
Test(
"rad10 with leading + sign",
b"0 + -42",
Expr::Int(-42),
None,
),
Test("rad10 with leading - sign", b"0 + +42", Expr::Int(42), None),
Test(
"flag char is string, not (invalid) int",
b"-x",
Expr::String("-x".into()),
None,
),
Test(
"keyword parameter is string",
b"--exact",
Expr::String("--exact".into()),
None,
),
Test(
"ranges or relative paths not confused for int",
b"./a/b",
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
Expr::GlobPattern("./a/b".into(), false),
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 16:09:50 +00:00
None,
),
Test(
"semver data not confused for int",
Require that values that look like numbers parse as numberlike (#8635) # Description Require that any value that looks like it might be a number (starts with a digit, or a '-' + digit, or a '+' + digits, or a special form float like `-inf`, `inf`, or `NaN`) must now be treated as a number-like value. Number-like syntax can only parse into number-like values. Number-like values include: durations, ints, floats, ranges, filesizes, binary data, etc. # User-Facing Changes BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE Just making sure we see this for release notes 😅 This breaks any and all numberlike values that were treated as strings before. Example, we used to allow `3,` as a bare word. Anything like this would now require quotes or backticks to be treated as a string or bare word, respectively. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-28 06:31:38 +00:00
b"'1.0.1'",
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 16:09:50 +00:00
Expr::String("1.0.1".into()),
None,
),
];
for test in tests {
test_int(test.0, test.1, test.2, test.3);
}
}
#[ignore]
#[test]
pub fn multi_test_parse_number() {
struct Test<'a>(&'a str, &'a [u8], Expr, Option<&'a str>);
// use test expression of form '0 + x' to force parse() to parse x as numeric.
// if expression were just 'x', parse() would try other items that would mask the error we're looking for.
let tests = vec![
Test("float decimal", b"0 + 43.5", Expr::Float(43.5), None),
//Test("float with leading + sign", b"0 + +41.7", Expr::Float(-41.7), None),
Test(
"float with leading - sign",
b"0 + -41.7",
Expr::Float(-41.7),
None,
),
Test(
"float scientific notation",
b"0 + 3e10",
Expr::Float(3.00e10),
None,
),
Test(
"float decimal literal invalid digits",
b"0 + .3foo",
Expr::Int(0),
Some("invalid digits"),
),
Test(
"float scientific notation literal invalid digits",
b"0 + 3e0faa",
Expr::Int(0),
Some("invalid digits"),
),
Test(
// odd that error is unsupportedOperation, but it does fail.
"decimal literal int 2 leading signs",
b"0 + --9",
Expr::Int(0),
Some("UnsupportedOperation"),
),
//Test(
// ".<string> should not be taken as float",
// b"abc + .foo",
// Expr::String("..".into()),
// None,
//),
];
for test in tests {
test_int(test.0, test.1, test.2, test.3);
}
}
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 16:09:50 +00:00
#[ignore]
#[test]
fn test_parse_any() {
let test = b"1..10";
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, test, true);
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 16:09:50 +00:00
match (block, working_set.parse_errors.first()) {
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 16:09:50 +00:00
(_, Some(e)) => {
println!("test: {test:?}, error: {e:#?}");
}
(b, None) => {
println!("test: {test:?}, parse: {b:#?}");
}
}
}
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
2021-08-30 18:36:07 +00:00
#[test]
pub fn parse_int() {
2021-09-02 08:25:22 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
2021-08-30 18:36:07 +00:00
let block = parse(&mut working_set, None, b"3", true);
2021-08-30 18:36:07 +00:00
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert_eq!(element.expr.expr, Expr::Int(3));
2021-08-30 18:36:07 +00:00
}
#[test]
pub fn parse_int_with_underscores() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"420_69_2023", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert_eq!(element.expr.expr, Expr::Int(420692023));
}
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
#[test]
pub fn parse_cell_path() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
working_set.add_variable(
"foo".to_string().into_bytes(),
Span::test_data(),
nu_protocol::Type::record(),
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
false,
);
let block = parse(&mut working_set, None, b"$foo.bar.baz", true);
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
assert!(working_set.parse_errors.is_empty());
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
if let Expr::FullCellPath(b) = &element.expr.expr {
assert!(matches!(b.head.expr, Expr::Var(_)));
if let [a, b] = &b.tail[..] {
if let PathMember::String { val, optional, .. } = a {
assert_eq!(val, "bar");
assert_eq!(optional, &false);
} else {
panic!("wrong type")
}
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
if let PathMember::String { val, optional, .. } = b {
assert_eq!(val, "baz");
assert_eq!(optional, &false);
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
} else {
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
panic!("wrong type")
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
}
} else {
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
panic!("cell path tail is unexpected")
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
}
} else {
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
panic!("Not a cell path");
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
}
}
#[test]
pub fn parse_cell_path_optional() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
working_set.add_variable(
"foo".to_string().into_bytes(),
Span::test_data(),
nu_protocol::Type::record(),
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
false,
);
let block = parse(&mut working_set, None, b"$foo.bar?.baz", true);
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
assert!(working_set.parse_errors.is_empty());
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
if let Expr::FullCellPath(b) = &element.expr.expr {
assert!(matches!(b.head.expr, Expr::Var(_)));
if let [a, b] = &b.tail[..] {
if let PathMember::String { val, optional, .. } = a {
assert_eq!(val, "bar");
assert_eq!(optional, &true);
} else {
panic!("wrong type")
}
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
if let PathMember::String { val, optional, .. } = b {
assert_eq!(val, "baz");
assert_eq!(optional, &false);
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
} else {
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
panic!("wrong type")
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
}
} else {
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
panic!("cell path tail is unexpected")
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
}
} else {
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
panic!("Not a cell path");
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 03:50:58 +00:00
}
}
#[test]
pub fn parse_binary_with_hex_format() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"0x[13]", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert_eq!(element.expr.expr, Expr::Binary(vec![0x13]));
}
#[test]
pub fn parse_binary_with_incomplete_hex_format() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"0x[3]", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert_eq!(element.expr.expr, Expr::Binary(vec![0x03]));
}
#[test]
pub fn parse_binary_with_binary_format() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"0b[1010 1000]", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert_eq!(element.expr.expr, Expr::Binary(vec![0b10101000]));
}
#[test]
pub fn parse_binary_with_incomplete_binary_format() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"0b[10]", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert_eq!(element.expr.expr, Expr::Binary(vec![0b00000010]));
}
#[test]
pub fn parse_binary_with_octal_format() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"0o[250]", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert_eq!(element.expr.expr, Expr::Binary(vec![0o250]));
}
#[test]
pub fn parse_binary_with_incomplete_octal_format() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"0o[2]", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert_eq!(element.expr.expr, Expr::Binary(vec![0o2]));
}
#[test]
pub fn parse_binary_with_invalid_octal_format() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"0b[90]", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert!(!matches!(element.expr.expr, Expr::Binary(_)));
}
2022-06-17 18:11:48 +00:00
#[test]
pub fn parse_binary_with_multi_byte_char() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
// found using fuzzing, Rust can panic if you slice into this string
let contents = b"0x[\xEF\xBF\xBD]";
let block = parse(&mut working_set, None, contents, true);
2022-06-17 18:11:48 +00:00
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert!(!matches!(element.expr.expr, Expr::Binary(_)))
2022-06-17 18:11:48 +00:00
}
2021-08-30 18:36:07 +00:00
#[test]
pub fn parse_call() {
2021-09-02 08:25:22 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
2021-08-30 18:36:07 +00:00
let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
2021-09-02 18:21:37 +00:00
working_set.add_decl(sig.predeclare());
2021-08-30 18:36:07 +00:00
let block = parse(&mut working_set, None, b"foo", true);
2021-08-30 18:36:07 +00:00
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
2021-08-30 18:36:07 +00:00
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
if let Expr::Call(call) = &element.expr.expr {
assert_eq!(call.decl_id, 0);
2021-08-30 18:36:07 +00:00
}
}
#[test]
pub fn parse_call_missing_flag_arg() {
2021-09-02 08:25:22 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
2021-08-30 18:36:07 +00:00
2021-10-13 17:53:27 +00:00
let sig = Signature::build("foo").named("jazz", SyntaxShape::Int, "jazz!!", Some('j'));
2021-09-02 18:21:37 +00:00
working_set.add_decl(sig.predeclare());
2021-08-30 18:36:07 +00:00
parse(&mut working_set, None, b"foo --jazz", true);
assert!(matches!(
working_set.parse_errors.first(),
Some(ParseError::MissingFlagParam(..))
));
2021-08-30 18:36:07 +00:00
}
#[test]
pub fn parse_call_missing_short_flag_arg() {
2021-09-02 08:25:22 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
2021-08-30 18:36:07 +00:00
let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
2021-09-02 18:21:37 +00:00
working_set.add_decl(sig.predeclare());
2021-08-30 18:36:07 +00:00
parse(&mut working_set, None, b"foo -j", true);
assert!(matches!(
working_set.parse_errors.first(),
Some(ParseError::MissingFlagParam(..))
));
2021-08-30 18:36:07 +00:00
}
#[test]
pub fn parse_call_short_flag_batch_arg_allowed() {
2021-09-02 08:25:22 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
2021-08-30 18:36:07 +00:00
let sig = Signature::build("foo")
.named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'))
.switch("--math", "math!!", Some('m'));
2021-09-02 18:21:37 +00:00
working_set.add_decl(sig.predeclare());
let block = parse(&mut working_set, None, b"foo -mj 10", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
if let Expr::Call(call) = &element.expr.expr {
assert_eq!(call.decl_id, 0);
Span ID Refactor (Step 2): Make Call SpanId-friendly (#13268) <!-- if this PR closes one or more issues, you can automatically link the PR with them by using one of the [*linking keywords*](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword), e.g. - this PR should close #xxxx - fixes #xxxx you can also mention related issues, PRs or discussions! --> # Description <!-- Thank you for improving Nushell. Please, check our [contributing guide](../CONTRIBUTING.md) and talk to the core team before making major changes. Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience. --> Part of https://github.com/nushell/nushell/issues/12963, step 2. This PR refactors Call and related argument structures to remove their dependency on `Expression::span` which will be removed in the future. # User-Facing Changes <!-- List of all changes that impact the user experience here. This helps us keep track of breaking changes. --> Should be none. If you see some error messages that look broken, please report. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass (on Windows make sure to [enable developer mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging)) - `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. -->
2024-07-03 06:00:52 +00:00
assert_eq!(call.arguments.item.len(), 2);
matches!(call.arguments.item[0], Argument::Named((_, None, None, _)));
matches!(
call.arguments.item[1],
Argument::Named((_, None, Some(_), _))
);
}
}
#[test]
pub fn parse_call_short_flag_batch_arg_disallowed() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let sig = Signature::build("foo")
.named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'))
.switch("--math", "math!!", Some('m'));
working_set.add_decl(sig.predeclare());
parse(&mut working_set, None, b"foo -jm 10", true);
assert!(matches!(
working_set.parse_errors.first(),
Some(ParseError::OnlyLastFlagInBatchCanTakeArg(..))
));
}
#[test]
pub fn parse_call_short_flag_batch_disallow_multiple_args() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let sig = Signature::build("foo")
.named("--math", SyntaxShape::Int, "math!!", Some('m'))
.named("--jazz", SyntaxShape::Int, "jazz!!", Some('j'));
working_set.add_decl(sig.predeclare());
parse(&mut working_set, None, b"foo -mj 10 20", true);
2021-08-30 18:36:07 +00:00
assert!(matches!(
working_set.parse_errors.first(),
Some(ParseError::OnlyLastFlagInBatchCanTakeArg(..))
2021-08-30 18:36:07 +00:00
));
}
#[test]
pub fn parse_call_unknown_shorthand() {
2021-09-02 08:25:22 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
2021-08-30 18:36:07 +00:00
let sig = Signature::build("foo").switch("--jazz", "jazz!!", Some('j'));
2021-09-02 18:21:37 +00:00
working_set.add_decl(sig.predeclare());
parse(&mut working_set, None, b"foo -mj", true);
assert!(matches!(
working_set.parse_errors.first(),
Some(ParseError::UnknownFlag(..))
));
2021-08-30 18:36:07 +00:00
}
#[test]
pub fn parse_call_extra_positional() {
2021-09-02 08:25:22 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
2021-08-30 18:36:07 +00:00
let sig = Signature::build("foo").switch("--jazz", "jazz!!", Some('j'));
2021-09-02 18:21:37 +00:00
working_set.add_decl(sig.predeclare());
parse(&mut working_set, None, b"foo -j 100", true);
assert!(matches!(
working_set.parse_errors.first(),
Some(ParseError::ExtraPositional(..))
));
2021-08-30 18:36:07 +00:00
}
#[test]
pub fn parse_call_missing_req_positional() {
2021-09-02 08:25:22 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
2021-08-30 18:36:07 +00:00
let sig = Signature::build("foo").required("jazz", SyntaxShape::Int, "jazz!!");
2021-09-02 18:21:37 +00:00
working_set.add_decl(sig.predeclare());
parse(&mut working_set, None, b"foo", true);
assert!(matches!(
working_set.parse_errors.first(),
Some(ParseError::MissingPositional(..))
));
2021-08-30 18:36:07 +00:00
}
#[test]
pub fn parse_call_missing_req_flag() {
2021-09-02 08:25:22 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
2021-08-30 18:36:07 +00:00
let sig = Signature::build("foo").required_named("--jazz", SyntaxShape::Int, "jazz!!", None);
2021-09-02 18:21:37 +00:00
working_set.add_decl(sig.predeclare());
parse(&mut working_set, None, b"foo", true);
assert!(matches!(
working_set.parse_errors.first(),
Some(ParseError::MissingRequiredFlag(..))
));
2021-08-30 18:36:07 +00:00
}
2021-09-04 22:25:31 +00:00
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
fn test_external_call(input: &str, tag: &str, f: impl FnOnce(&Expression, &[ExternalArgument])) {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, input.as_bytes(), true);
assert!(
working_set.parse_errors.is_empty(),
"{tag}: errors: {:?}",
working_set.parse_errors
);
let pipeline = &block.pipelines[0];
assert_eq!(1, pipeline.len());
let element = &pipeline.elements[0];
match &element.expr.expr {
Expr::ExternalCall(name, args) => f(name, args),
other => {
panic!("{tag}: Unexpected expression in pipeline: {other:?}");
}
}
}
fn check_external_call_interpolation(
tag: &str,
subexpr_count: usize,
quoted: bool,
expr: &Expression,
) -> bool {
match &expr.expr {
Expr::StringInterpolation(exprs) => {
assert!(quoted, "{tag}: quoted");
assert_eq!(expr.ty, Type::String, "{tag}: expr.ty");
assert_eq!(subexpr_count, exprs.len(), "{tag}: subexpr_count");
true
}
Expr::GlobInterpolation(exprs, is_quoted) => {
assert_eq!(quoted, *is_quoted, "{tag}: quoted");
assert_eq!(expr.ty, Type::Glob, "{tag}: expr.ty");
assert_eq!(subexpr_count, exprs.len(), "{tag}: subexpr_count");
true
}
_ => false,
}
}
#[rstest]
#[case("foo-external-call", "foo-external-call", "bare word")]
#[case("^foo-external-call", "foo-external-call", "bare word with caret")]
#[case(
"foo/external-call",
"foo/external-call",
"bare word with forward slash"
)]
#[case(
"^foo/external-call",
"foo/external-call",
"bare word with forward slash and caret"
)]
#[case(r"foo\external-call", r"foo\external-call", "bare word with backslash")]
#[case(
r"^foo\external-call",
r"foo\external-call",
"bare word with backslash and caret"
)]
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
#[case("`foo external call`", "foo external call", "backtick quote")]
#[case(
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
"^`foo external call`",
"foo external call",
"backtick quote with caret"
)]
#[case(
"`foo/external call`",
"foo/external call",
"backtick quote with forward slash"
)]
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
#[case(
"^`foo/external call`",
"foo/external call",
"backtick quote with forward slash and caret"
)]
#[case(
r"`foo\external call`",
r"foo\external call",
"backtick quote with backslash"
)]
#[case(
r"^`foo\external call`",
r"foo\external call",
"backtick quote with backslash and caret"
)]
pub fn test_external_call_head_glob(
#[case] input: &str,
#[case] expected: &str,
#[case] tag: &str,
) {
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::GlobPattern(string, is_quoted) => {
assert_eq!(expected, string, "{tag}: incorrect name");
assert!(!*is_quoted);
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(0, args.len());
})
}
#[rstest]
#[case(
r##"^r#'foo-external-call'#"##,
"foo-external-call",
"raw string with caret"
)]
#[case(
r##"^r#'foo/external-call'#"##,
"foo/external-call",
"raw string with forward slash and caret"
)]
#[case(
r##"^r#'foo\external-call'#"##,
r"foo\external-call",
"raw string with backslash and caret"
)]
pub fn test_external_call_head_raw_string(
#[case] input: &str,
#[case] expected: &str,
#[case] tag: &str,
) {
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::RawString(string) => {
assert_eq!(expected, string, "{tag}: incorrect name");
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(0, args.len());
})
}
#[rstest]
#[case("^'foo external call'", "foo external call", "single quote with caret")]
#[case(
"^'foo/external call'",
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
"foo/external call",
"single quote with forward slash and caret"
)]
#[case(
r"^'foo\external call'",
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
r"foo\external call",
"single quote with backslash and caret"
)]
#[case(
r#"^"foo external call""#,
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
r#"foo external call"#,
"double quote with caret"
)]
#[case(
r#"^"foo/external call""#,
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
r#"foo/external call"#,
"double quote with forward slash and caret"
)]
#[case(
r#"^"foo\\external call""#,
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
r#"foo\external call"#,
"double quote with backslash and caret"
)]
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
pub fn test_external_call_head_string(
#[case] input: &str,
#[case] expected: &str,
#[case] tag: &str,
) {
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::String(string) => {
assert_eq!(expected, string);
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(0, args.len());
})
}
#[rstest]
#[case(r"~/.foo/(1)", 2, false, "unquoted interpolated string")]
#[case(
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
r"~\.foo(2)\(1)",
4,
false,
"unquoted interpolated string with backslash"
)]
#[case(r"^~/.foo/(1)", 2, false, "unquoted interpolated string with caret")]
#[case(r#"^$"~/.foo/(1)""#, 2, true, "quoted interpolated string with caret")]
pub fn test_external_call_head_interpolated_string(
#[case] input: &str,
#[case] subexpr_count: usize,
#[case] quoted: bool,
#[case] tag: &str,
) {
test_external_call(input, tag, |name, args| {
if !check_external_call_interpolation(tag, subexpr_count, quoted, name) {
panic!("{tag}: Unexpected expression in command name position: {name:?}");
}
assert_eq!(0, args.len());
})
}
#[rstest]
#[case("^foo foo-external-call", "foo-external-call", "bare word")]
#[case(
"^foo foo/external-call",
"foo/external-call",
"bare word with forward slash"
)]
#[case(
r"^foo foo\external-call",
r"foo\external-call",
"bare word with backslash"
)]
#[case(
"^foo `foo external call`",
"foo external call",
"backtick quote with caret"
)]
#[case(
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
"^foo `foo/external call`",
"foo/external call",
"backtick quote with forward slash"
)]
#[case(
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
r"^foo `foo\external call`",
r"foo\external call",
"backtick quote with backslash"
)]
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
pub fn test_external_call_arg_glob(#[case] input: &str, #[case] expected: &str, #[case] tag: &str) {
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::GlobPattern(string, _) => {
assert_eq!("foo", string, "{tag}: incorrect name");
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(1, args.len());
match &args[0] {
ExternalArgument::Regular(expr) => match &expr.expr {
Expr::GlobPattern(string, is_quoted) => {
assert_eq!(expected, string, "{tag}: incorrect arg");
assert!(!*is_quoted);
}
other => {
panic!("Unexpected expression in command arg position: {other:?}")
}
},
other @ ExternalArgument::Spread(..) => {
panic!("Unexpected external spread argument in command arg position: {other:?}")
}
}
})
}
#[rstest]
#[case(r##"^foo r#'foo-external-call'#"##, "foo-external-call", "raw string")]
#[case(
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
r##"^foo r#'foo/external-call'#"##,
"foo/external-call",
"raw string with forward slash"
)]
#[case(
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
r##"^foo r#'foo\external-call'#"##,
r"foo\external-call",
"raw string with backslash"
)]
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
pub fn test_external_call_arg_raw_string(
#[case] input: &str,
#[case] expected: &str,
#[case] tag: &str,
) {
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::GlobPattern(string, _) => {
assert_eq!("foo", string, "{tag}: incorrect name");
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(1, args.len());
match &args[0] {
ExternalArgument::Regular(expr) => match &expr.expr {
Expr::RawString(string) => {
assert_eq!(expected, string, "{tag}: incorrect arg");
}
other => {
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
panic!("Unexpected expression in command arg position: {other:?}")
}
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
},
other @ ExternalArgument::Spread(..) => {
panic!("Unexpected external spread argument in command arg position: {other:?}")
}
}
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
})
}
#[rstest]
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
#[case("^foo 'foo external call'", "foo external call", "single quote")]
#[case(
"^foo 'foo/external call'",
"foo/external call",
"single quote with forward slash"
)]
#[case(
r"^foo 'foo\external call'",
r"foo\external call",
"single quote with backslash"
)]
#[case(r#"^foo "foo external call""#, r#"foo external call"#, "double quote")]
#[case(
r#"^foo "foo/external call""#,
r#"foo/external call"#,
"double quote with forward slash"
)]
#[case(
r#"^foo "foo\\external call""#,
r#"foo\external call"#,
"double quote with backslash"
)]
pub fn test_external_call_arg_string(
#[case] input: &str,
#[case] expected: &str,
#[case] tag: &str,
) {
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::GlobPattern(string, _) => {
assert_eq!("foo", string, "{tag}: incorrect name");
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(1, args.len());
match &args[0] {
ExternalArgument::Regular(expr) => match &expr.expr {
Expr::String(string) => {
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
assert_eq!(expected, string, "{tag}: incorrect arg");
}
other => {
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
panic!("{tag}: Unexpected expression in command arg position: {other:?}")
}
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
},
other @ ExternalArgument::Spread(..) => {
panic!(
"{tag}: Unexpected external spread argument in command arg position: {other:?}"
)
}
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
}
})
}
#[rstest]
#[case(r"^foo ~/.foo/(1)", 2, false, "unquoted interpolated string")]
#[case(r#"^foo $"~/.foo/(1)""#, 2, true, "quoted interpolated string")]
pub fn test_external_call_arg_interpolated_string(
#[case] input: &str,
#[case] subexpr_count: usize,
#[case] quoted: bool,
#[case] tag: &str,
) {
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::GlobPattern(string, _) => {
assert_eq!("foo", string, "{tag}: incorrect name");
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
assert_eq!(1, args.len());
match &args[0] {
ExternalArgument::Regular(expr) => {
if !check_external_call_interpolation(tag, subexpr_count, quoted, expr) {
panic!("Unexpected expression in command arg position: {expr:?}")
}
}
other @ ExternalArgument::Spread(..) => {
panic!("Unexpected external spread argument in command arg position: {other:?}")
}
}
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
})
}
#[test]
fn test_external_call_argument_spread() {
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
let input = r"^foo ...[a b c]";
let tag = "spread";
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::GlobPattern(string, _) => {
assert_eq!("foo", string, "incorrect name");
}
other => {
panic!("Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(1, args.len());
match &args[0] {
ExternalArgument::Spread(expr) => match &expr.expr {
Expr::List(items) => {
assert_eq!(3, items.len());
// that's good enough, don't really need to go so deep into it...
}
other => {
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
panic!("Unexpected expression in command arg position: {other:?}")
}
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
},
other @ ExternalArgument::Regular(..) => {
panic!("Unexpected external regular argument in command arg position: {other:?}")
}
}
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
})
}
#[test]
fn test_nothing_comparison_eq() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"2 == null", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert!(matches!(&element.expr.expr, Expr::BinaryOp(..)));
}
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
#[rstest]
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
#[case(b"let a = 1 err> /dev/null")]
#[case(b"let a = 1 out> /dev/null")]
#[case(b"mut a = 1 err> /dev/null")]
#[case(b"mut a = 1 out> /dev/null")]
#[case(b"let a = 1 out+err> /dev/null")]
#[case(b"mut a = 1 out+err> /dev/null")]
fn test_redirection_with_letmut(#[case] phase: &[u8]) {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let _block = parse(&mut working_set, None, phase, true);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
assert!(matches!(
working_set.parse_errors.first(),
Some(ParseError::RedirectingBuiltinCommand(_, _, _))
));
}
#[rstest]
#[case(b"o>")]
#[case(b"o>>")]
#[case(b"e>")]
#[case(b"e>>")]
#[case(b"o+e>")]
#[case(b"o+e>>")]
#[case(b"e>|")]
#[case(b"o+e>|")]
#[case(b"|o>")]
#[case(b"|o>>")]
#[case(b"|e>")]
#[case(b"|e>>")]
#[case(b"|o+e>")]
#[case(b"|o+e>>")]
#[case(b"|e>|")]
#[case(b"|o+e>|")]
#[case(b"e> file")]
#[case(b"e>> file")]
#[case(b"o> file")]
#[case(b"o>> file")]
#[case(b"o+e> file")]
#[case(b"o+e>> file")]
#[case(b"|e> file")]
#[case(b"|e>> file")]
#[case(b"|o> file")]
#[case(b"|o>> file")]
#[case(b"|o+e> file")]
#[case(b"|o+e>> file")]
fn test_redirecting_nothing(#[case] text: &[u8]) {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let _ = parse(&mut working_set, None, text, true);
assert!(matches!(
working_set.parse_errors.first(),
Some(ParseError::UnexpectedRedirection { .. })
));
}
#[test]
fn test_nothing_comparison_neq() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"2 != null", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert!(matches!(&element.expr.expr, Expr::BinaryOp(..)));
}
mod string {
use super::*;
#[test]
pub fn parse_string() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"\"hello nushell\"", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
assert_eq!(element.expr.expr, Expr::String("hello nushell".to_string()))
}
mod interpolation {
use nu_protocol::Span;
use super::*;
#[test]
pub fn parse_string_interpolation() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"$\"hello (39 + 3)\"", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let subexprs: Vec<&Expr> = match &element.expr.expr {
Expr::StringInterpolation(expressions) => {
expressions.iter().map(|e| &e.expr).collect()
}
_ => panic!("Expected an `Expr::StringInterpolation`"),
};
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
assert_eq!(subexprs.len(), 2);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
assert_eq!(subexprs[0], &Expr::String("hello ".to_string()));
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
assert!(matches!(subexprs[1], &Expr::FullCellPath(..)));
}
#[test]
pub fn parse_string_interpolation_escaped_parenthesis() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"$\"hello \\(39 + 3)\"", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
let subexprs: Vec<&Expr> = match &element.expr.expr {
Expr::StringInterpolation(expressions) => {
expressions.iter().map(|e| &e.expr).collect()
}
_ => panic!("Expected an `Expr::StringInterpolation`"),
};
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
assert_eq!(subexprs.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
assert_eq!(subexprs[0], &Expr::String("hello (39 + 3)".to_string()));
}
#[test]
pub fn parse_string_interpolation_escaped_backslash_before_parenthesis() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"$\"hello \\\\(39 + 3)\"", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
let subexprs: Vec<&Expr> = match &element.expr.expr {
Expr::StringInterpolation(expressions) => {
expressions.iter().map(|e| &e.expr).collect()
}
_ => panic!("Expected an `Expr::StringInterpolation`"),
};
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
assert_eq!(subexprs.len(), 2);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
assert_eq!(subexprs[0], &Expr::String("hello \\".to_string()));
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
assert!(matches!(subexprs[1], &Expr::FullCellPath(..)));
}
#[test]
pub fn parse_string_interpolation_backslash_count_reset_by_expression() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"$\"\\(1 + 3)\\(7 - 5)\"", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
let subexprs: Vec<&Expr> = match &element.expr.expr {
Expr::StringInterpolation(expressions) => {
expressions.iter().map(|e| &e.expr).collect()
}
_ => panic!("Expected an `Expr::StringInterpolation`"),
};
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
assert_eq!(subexprs.len(), 1);
assert_eq!(subexprs[0], &Expr::String("(1 + 3)(7 - 5)".to_string()));
}
Move most of the peculiar argument handling for external calls into the parser (#13089) # Description We've had a lot of different issues and PRs related to arg handling with externals since the rewrite of `run-external` in #12921: - #12950 - #12955 - #13000 - #13001 - #13021 - #13027 - #13028 - #13073 Many of these are caused by the argument handling of external calls and `run-external` being very special and involving the parser handing quoted strings over to `run-external` so that it knows whether to expand tildes and globs and so on. This is really unusual and also makes it harder to use `run-external`, and also harder to understand it (and probably is part of the reason why it was rewritten in the first place). This PR moves a lot more of that work over to the parser, so that by the time `run-external` gets it, it's dealing with much more normal Nushell values. In particular: - Unquoted strings are handled as globs with no expand - The unescaped-but-quoted handling of strings was removed, and the parser constructs normal looking strings instead, removing internal quotes so that `run-external` doesn't have to do it - Bare word interpolation is now supported and expansion is done in this case - Expressions typed as `Glob` containing `Expr::StringInterpolation` now produce `Value::Glob` instead, with the quoted status from the expr passed through so we know if it was a bare word - Bare word interpolation for values typed as `glob` now possible, but not implemented - Because expansion is now triggered by `Value::Glob(_, false)` instead of looking at the expr, externals now support glob types # User-Facing Changes - Bare word interpolation works for external command options, and otherwise embedded in other strings: ```nushell ^echo --foo=(2 + 2) # prints --foo=4 ^echo -foo=$"(2 + 2)" # prints -foo=4 ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2) ^echo foo,(2 + 2),bar # prints foo,4,bar ``` - Bare word interpolation expands for external command head/args: ```nushell let name = "exa" ~/.cargo/bin/($name) # this works, and expands the tilde ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde ^echo ~/($name)/* # this glob is expanded ^echo $"~/($name)/*" # this isn't expanded ``` - Ndots are now supported for the head of an external command (`^.../foo` works) - Glob values are now supported for head/args of an external command, and expanded appropriately: ```nushell ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded ^echo ("*.txt" | into glob) # this glob is expanded ``` - `run-external` now works more like any other command, without expecting a special call convention for its args: ```nushell run-external echo "'foo'" # before PR: 'foo' # after PR: foo run-external echo "*.txt" # before PR: (glob is expanded) # after PR: *.txt ``` # Tests + Formatting Lots of tests added and cleaned up. Some tests that weren't active on Windows changed to use `nu --testbin cococo` so that they can work. Added a test for Linux only to make sure tilde expansion of commands works, because changing `HOME` there causes `~` to reliably change. - :green_circle: `toolkit fmt` - :green_circle: `toolkit clippy` - :green_circle: `toolkit test` - :green_circle: `toolkit test stdlib` # After Submitting - [ ] release notes: make sure to mention the new syntaxes that are supported
2024-06-20 04:00:03 +00:00
#[test]
pub fn parse_string_interpolation_bare() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(
&mut working_set,
None,
b"\"\" ++ foo(1 + 3)bar(7 - 5)",
true,
);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
let subexprs: Vec<&Expr> = match &element.expr.expr {
Expr::BinaryOp(_, _, rhs) => match &rhs.expr {
Expr::StringInterpolation(expressions) => {
expressions.iter().map(|e| &e.expr).collect()
}
_ => panic!("Expected an `Expr::StringInterpolation`"),
},
_ => panic!("Expected an `Expr::BinaryOp`"),
};
assert_eq!(subexprs.len(), 4);
assert_eq!(subexprs[0], &Expr::String("foo".to_string()));
assert!(matches!(subexprs[1], &Expr::FullCellPath(..)));
assert_eq!(subexprs[2], &Expr::String("bar".to_string()));
assert!(matches!(subexprs[3], &Expr::FullCellPath(..)));
}
#[test]
pub fn parse_nested_expressions() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
working_set.add_variable(
"foo".to_string().into_bytes(),
Span::new(0, 0),
nu_protocol::Type::CellPath,
false,
);
parse(
&mut working_set,
None,
br#"
$"(($foo))"
"#,
true,
);
assert!(working_set.parse_errors.is_empty());
}
#[test]
pub fn parse_path_expression() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
working_set.add_variable(
"foo".to_string().into_bytes(),
Span::new(0, 0),
nu_protocol::Type::CellPath,
false,
);
parse(
&mut working_set,
None,
br#"
$"Hello ($foo.bar)"
"#,
true,
);
assert!(working_set.parse_errors.is_empty());
}
}
#[test]
fn parse_raw_string_as_external_argument() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"^echo r#'text'#", true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
if let Expr::ExternalCall(_, args) = &element.expr.expr {
if let [ExternalArgument::Regular(expr)] = args.as_ref() {
assert_eq!(expr.expr, Expr::RawString("text".into()));
return;
}
}
panic!("wrong expression: {:?}", element.expr.expr)
}
}
#[rstest]
#[case(b"let a = }")]
#[case(b"mut a = }")]
#[case(b"let a = | }")]
#[case(b"mut a = | }")]
fn test_semi_open_brace(#[case] phrase: &[u8]) {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
// this should not panic
let _block = parse(&mut working_set, None, phrase, true);
}
2021-09-04 22:25:31 +00:00
mod range {
use super::*;
use nu_protocol::ast::{RangeInclusion, RangeOperator};
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
#[rstest]
#[case(b"0..10", RangeInclusion::Inclusive, "inclusive")]
#[case(b"0..=10", RangeInclusion::Inclusive, "=inclusive")]
#[case(b"0..<10", RangeInclusion::RightExclusive, "exclusive")]
#[case(b"10..0", RangeInclusion::Inclusive, "reverse inclusive")]
#[case(b"10..=0", RangeInclusion::Inclusive, "reverse =inclusive")]
#[case(
b"(3 - 3)..<(8 + 2)",
RangeInclusion::RightExclusive,
"subexpression exclusive"
)]
#[case(
b"(3 - 3)..(8 + 2)",
RangeInclusion::Inclusive,
"subexpression inclusive"
)]
#[case(
b"(3 - 3)..=(8 + 2)",
RangeInclusion::Inclusive,
"subexpression =inclusive"
)]
#[case(b"-10..-3", RangeInclusion::Inclusive, "negative inclusive")]
#[case(b"-10..=-3", RangeInclusion::Inclusive, "negative =inclusive")]
#[case(b"-10..<-3", RangeInclusion::RightExclusive, "negative exclusive")]
fn parse_bounded_range(
#[case] phrase: &[u8],
#[case] inclusion: RangeInclusion,
#[case] tag: &str,
) {
2021-09-04 22:25:31 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, phrase, true);
2021-09-04 22:25:31 +00:00
assert!(working_set.parse_errors.is_empty());
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
assert_eq!(block.len(), 1, "{tag}: block length");
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1, "{tag}: expression length");
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
if let Expr::Range(range) = &element.expr.expr {
if let Range {
from: Some(_),
next: None,
to: Some(_),
operator:
RangeOperator {
inclusion: the_inclusion,
..
},
} = range.as_ref()
{
assert_eq!(
*the_inclusion, inclusion,
"{tag}: wrong RangeInclusion {the_inclusion:?}"
);
} else {
panic!("{tag}: expression mismatch.")
}
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
} else {
panic!("{tag}: expression mismatch.")
};
2021-09-04 22:25:31 +00:00
}
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
#[rstest]
#[case(
b"let a = 2; $a..10",
RangeInclusion::Inclusive,
"variable start inclusive"
)]
#[case(
b"let a = 2; $a..=10",
RangeInclusion::Inclusive,
"variable start =inclusive"
)]
#[case(
b"let a = 2; $a..<($a + 10)",
RangeInclusion::RightExclusive,
"subexpression variable exclusive"
)]
fn parse_variable_range(
#[case] phrase: &[u8],
#[case] inclusion: RangeInclusion,
#[case] tag: &str,
) {
2021-09-04 22:25:31 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
2021-09-13 08:19:05 +00:00
working_set.add_decl(Box::new(Let));
let block = parse(&mut working_set, None, phrase, true);
2021-09-04 22:25:31 +00:00
assert!(working_set.parse_errors.is_empty());
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
assert_eq!(block.len(), 2, "{tag} block len 2");
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[1];
assert_eq!(pipeline.len(), 1, "{tag}: expression length 1");
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
if let Expr::Range(range) = &element.expr.expr {
if let Range {
from: Some(_),
next: None,
to: Some(_),
operator:
RangeOperator {
inclusion: the_inclusion,
..
},
} = range.as_ref()
{
assert_eq!(
*the_inclusion, inclusion,
"{tag}: wrong RangeInclusion {the_inclusion:?}"
);
} else {
panic!("{tag}: expression mismatch.")
}
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
} else {
panic!("{tag}: expression mismatch.")
};
2021-09-04 22:25:31 +00:00
}
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
#[rstest]
#[case(b"0..", RangeInclusion::Inclusive, "right unbounded")]
#[case(b"0..=", RangeInclusion::Inclusive, "right unbounded =inclusive")]
#[case(b"0..<", RangeInclusion::RightExclusive, "right unbounded")]
2021-09-04 22:25:31 +00:00
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
fn parse_right_unbounded_range(
#[case] phrase: &[u8],
#[case] inclusion: RangeInclusion,
#[case] tag: &str,
) {
2021-09-04 22:25:31 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, phrase, true);
2021-09-04 22:25:31 +00:00
assert!(working_set.parse_errors.is_empty());
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
assert_eq!(block.len(), 1, "{tag}: block len 1");
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1, "{tag}: expression length");
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
if let Expr::Range(range) = &element.expr.expr {
if let Range {
from: Some(_),
next: None,
to: None,
operator:
RangeOperator {
inclusion: the_inclusion,
..
},
} = range.as_ref()
{
assert_eq!(
*the_inclusion, inclusion,
"{tag}: wrong RangeInclusion {the_inclusion:?}"
);
} else {
panic!("{tag}: expression mismatch.")
}
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
} else {
panic!("{tag}: expression mismatch.")
};
2021-09-04 22:25:31 +00:00
}
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
#[rstest]
#[case(b"..10", RangeInclusion::Inclusive, "left unbounded inclusive")]
#[case(b"..=10", RangeInclusion::Inclusive, "left unbounded =inclusive")]
#[case(b"..<10", RangeInclusion::RightExclusive, "left unbounded exclusive")]
fn parse_left_unbounded_range(
#[case] phrase: &[u8],
#[case] inclusion: RangeInclusion,
#[case] tag: &str,
) {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, phrase, true);
assert!(working_set.parse_errors.is_empty());
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
assert_eq!(block.len(), 1, "{tag}: block len 1");
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1, "{tag}: expression length");
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
if let Expr::Range(range) = &element.expr.expr {
if let Range {
from: None,
next: None,
to: Some(_),
operator:
RangeOperator {
inclusion: the_inclusion,
..
},
} = range.as_ref()
{
assert_eq!(
*the_inclusion, inclusion,
"{tag}: wrong RangeInclusion {the_inclusion:?}"
);
} else {
panic!("{tag}: expression mismatch.")
}
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
} else {
panic!("{tag}: expression mismatch.")
};
}
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
#[rstest]
#[case(b"2.0..4.0..10.0", RangeInclusion::Inclusive, "float inclusive")]
#[case(b"2.0..4.0..=10.0", RangeInclusion::Inclusive, "float =inclusive")]
#[case(b"2.0..4.0..<10.0", RangeInclusion::RightExclusive, "float exclusive")]
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
fn parse_float_range(
#[case] phrase: &[u8],
#[case] inclusion: RangeInclusion,
#[case] tag: &str,
) {
2021-09-12 12:36:54 +00:00
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, phrase, true);
2021-09-12 12:36:54 +00:00
assert!(working_set.parse_errors.is_empty());
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
assert_eq!(block.len(), 1, "{tag}: block length 1");
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1, "{tag}: expression length");
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
if let Expr::Range(range) = &element.expr.expr {
if let Range {
from: Some(_),
next: Some(_),
to: Some(_),
operator:
RangeOperator {
inclusion: the_inclusion,
..
},
} = range.as_ref()
{
assert_eq!(
*the_inclusion, inclusion,
"{tag}: wrong RangeInclusion {the_inclusion:?}"
);
} else {
panic!("{tag}: expression mismatch.")
}
range operator accepts bot..=top as well as bot..top (#8382) # Description A compromise fix for #8162. Nushell range operator now accepts `..=` to mean the range includes the top value, so you can use your Rust habits. But the unadorned `..` range operator also includes the value, so you can also use your Nushell habits. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```nushell 〉1..5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..=5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ │ 4 │ 5 │ ╰───┴───╯ -------------------------------------------- /home/bobhy/src/rust/nushell -------------------------------------------- 〉1..<5 ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ 3 │ │ 3 │ 4 │ ╰───┴───╯ ``` # User-Facing Changes Existing scripts with range operator will continue to operate as heretofore. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting Will update the book to include new syntax.
2023-04-07 11:40:05 +00:00
} else {
panic!("{tag}: expression mismatch.")
};
2021-09-12 12:36:54 +00:00
}
#[test]
2021-09-05 18:44:18 +00:00
fn bad_parse_does_crash() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let _ = parse(&mut working_set, None, b"(0)..\"a\"", true);
assert!(!working_set.parse_errors.is_empty());
}
#[test]
fn vars_not_read_as_units() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let _ = parse(&mut working_set, None, b"0..<$day", true);
assert!(working_set.parse_errors.is_empty());
}
2021-09-04 22:25:31 +00:00
}
#[cfg(test)]
mod input_types {
use super::*;
use nu_protocol::{
ast::{Argument, Call},
Category, PipelineData, ShellError, Type,
};
#[derive(Clone)]
pub struct LsTest;
impl Command for LsTest {
fn name(&self) -> &str {
"ls"
}
fn usage(&self) -> &str {
"Mock ls command."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build(self.name()).category(Category::Default)
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
_call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
todo!()
}
}
#[derive(Clone)]
pub struct Def;
impl Command for Def {
fn name(&self) -> &str {
"def"
}
fn usage(&self) -> &str {
"Mock def command."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build("def")
.input_output_types(vec![(Type::Nothing, Type::Nothing)])
.required("def_name", SyntaxShape::String, "definition name")
.required("params", SyntaxShape::Signature, "parameters")
.required("body", SyntaxShape::Closure(None), "body of the definition")
.category(Category::Core)
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
_call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
todo!()
}
}
#[derive(Clone)]
pub struct GroupBy;
impl Command for GroupBy {
fn name(&self) -> &str {
"group-by"
}
fn usage(&self) -> &str {
"Mock group-by command."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build(self.name())
.required("column", SyntaxShape::String, "column name")
.category(Category::Default)
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
_call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
todo!()
}
}
#[derive(Clone)]
pub struct ToCustom;
impl Command for ToCustom {
fn name(&self) -> &str {
"to-custom"
}
fn usage(&self) -> &str {
"Mock converter command."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build(self.name())
Improve type hovers (#9515) # Description This PR does a few things to help improve type hovers and, in the process, fixes a few outstanding issues in the type system. Here's a list of the changes: * `for` now will try to infer the type of the iteration variable based on the expression it's given. This fixes things like `for x in [1, 2, 3] { }` where `x` now properly gets the int type. * Removed old input/output type fields from the signature, focuses on the vec of signatures. Updated a bunch of dataframe commands that hadn't moved over. This helps tie things together a bit better * Fixed inference of types from subexpressions to use the last expression in the block * Fixed handling of explicit types in `let` and `mut` calls, so we now respect that as the authoritative type I also tried to add `def` input/output type inference, but unfortunately we only know the predecl types universally, which means we won't have enough information to properly know what the types of the custom commands are. # User-Facing Changes Script typechecking will get tighter in some cases Hovers should be more accurate in some cases that previously resorted to any. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect -A clippy::result_large_err` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- crates/nu-std/tests/run.nu` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --> --------- Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com>
2023-06-28 17:19:48 +00:00
.input_output_type(Type::Any, Type::Custom("custom".into()))
.category(Category::Custom("custom".into()))
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
_call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
todo!()
}
}
#[derive(Clone)]
pub struct GroupByCustom;
impl Command for GroupByCustom {
fn name(&self) -> &str {
"group-by"
}
fn usage(&self) -> &str {
"Mock custom group-by command."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build(self.name())
.required("column", SyntaxShape::String, "column name")
.required("other", SyntaxShape::String, "other value")
Improve type hovers (#9515) # Description This PR does a few things to help improve type hovers and, in the process, fixes a few outstanding issues in the type system. Here's a list of the changes: * `for` now will try to infer the type of the iteration variable based on the expression it's given. This fixes things like `for x in [1, 2, 3] { }` where `x` now properly gets the int type. * Removed old input/output type fields from the signature, focuses on the vec of signatures. Updated a bunch of dataframe commands that hadn't moved over. This helps tie things together a bit better * Fixed inference of types from subexpressions to use the last expression in the block * Fixed handling of explicit types in `let` and `mut` calls, so we now respect that as the authoritative type I also tried to add `def` input/output type inference, but unfortunately we only know the predecl types universally, which means we won't have enough information to properly know what the types of the custom commands are. # User-Facing Changes Script typechecking will get tighter in some cases Hovers should be more accurate in some cases that previously resorted to any. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect -A clippy::result_large_err` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- crates/nu-std/tests/run.nu` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --> --------- Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com>
2023-06-28 17:19:48 +00:00
.input_output_type(Type::Custom("custom".into()), Type::Custom("custom".into()))
.category(Category::Custom("custom".into()))
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
_call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
todo!()
}
}
#[derive(Clone)]
pub struct AggCustom;
impl Command for AggCustom {
fn name(&self) -> &str {
"agg"
}
fn usage(&self) -> &str {
"Mock custom agg command."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build(self.name())
.required("operation", SyntaxShape::String, "operation")
Improve type hovers (#9515) # Description This PR does a few things to help improve type hovers and, in the process, fixes a few outstanding issues in the type system. Here's a list of the changes: * `for` now will try to infer the type of the iteration variable based on the expression it's given. This fixes things like `for x in [1, 2, 3] { }` where `x` now properly gets the int type. * Removed old input/output type fields from the signature, focuses on the vec of signatures. Updated a bunch of dataframe commands that hadn't moved over. This helps tie things together a bit better * Fixed inference of types from subexpressions to use the last expression in the block * Fixed handling of explicit types in `let` and `mut` calls, so we now respect that as the authoritative type I also tried to add `def` input/output type inference, but unfortunately we only know the predecl types universally, which means we won't have enough information to properly know what the types of the custom commands are. # User-Facing Changes Script typechecking will get tighter in some cases Hovers should be more accurate in some cases that previously resorted to any. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect -A clippy::result_large_err` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- crates/nu-std/tests/run.nu` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --> --------- Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com>
2023-06-28 17:19:48 +00:00
.input_output_type(Type::Custom("custom".into()), Type::Custom("custom".into()))
.category(Category::Custom("custom".into()))
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
_call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
todo!()
}
}
#[derive(Clone)]
pub struct AggMin;
impl Command for AggMin {
fn name(&self) -> &str {
"min"
}
fn usage(&self) -> &str {
"Mock custom min command."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build(self.name()).category(Category::Custom("custom".into()))
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
_call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
todo!()
}
}
#[derive(Clone)]
pub struct WithColumn;
impl Command for WithColumn {
fn name(&self) -> &str {
"with-column"
}
fn usage(&self) -> &str {
"Mock custom with-column command."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build(self.name())
.rest("operation", SyntaxShape::Any, "operation")
Improve type hovers (#9515) # Description This PR does a few things to help improve type hovers and, in the process, fixes a few outstanding issues in the type system. Here's a list of the changes: * `for` now will try to infer the type of the iteration variable based on the expression it's given. This fixes things like `for x in [1, 2, 3] { }` where `x` now properly gets the int type. * Removed old input/output type fields from the signature, focuses on the vec of signatures. Updated a bunch of dataframe commands that hadn't moved over. This helps tie things together a bit better * Fixed inference of types from subexpressions to use the last expression in the block * Fixed handling of explicit types in `let` and `mut` calls, so we now respect that as the authoritative type I also tried to add `def` input/output type inference, but unfortunately we only know the predecl types universally, which means we won't have enough information to properly know what the types of the custom commands are. # User-Facing Changes Script typechecking will get tighter in some cases Hovers should be more accurate in some cases that previously resorted to any. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect -A clippy::result_large_err` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- crates/nu-std/tests/run.nu` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --> --------- Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com>
2023-06-28 17:19:48 +00:00
.input_output_type(Type::Custom("custom".into()), Type::Custom("custom".into()))
.category(Category::Custom("custom".into()))
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
_call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
todo!()
}
}
#[derive(Clone)]
pub struct Collect;
impl Command for Collect {
fn name(&self) -> &str {
"collect"
}
fn usage(&self) -> &str {
"Mock custom collect command."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build(self.name())
Improve type hovers (#9515) # Description This PR does a few things to help improve type hovers and, in the process, fixes a few outstanding issues in the type system. Here's a list of the changes: * `for` now will try to infer the type of the iteration variable based on the expression it's given. This fixes things like `for x in [1, 2, 3] { }` where `x` now properly gets the int type. * Removed old input/output type fields from the signature, focuses on the vec of signatures. Updated a bunch of dataframe commands that hadn't moved over. This helps tie things together a bit better * Fixed inference of types from subexpressions to use the last expression in the block * Fixed handling of explicit types in `let` and `mut` calls, so we now respect that as the authoritative type I also tried to add `def` input/output type inference, but unfortunately we only know the predecl types universally, which means we won't have enough information to properly know what the types of the custom commands are. # User-Facing Changes Script typechecking will get tighter in some cases Hovers should be more accurate in some cases that previously resorted to any. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect -A clippy::result_large_err` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- crates/nu-std/tests/run.nu` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --> --------- Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com>
2023-06-28 17:19:48 +00:00
.input_output_type(Type::Custom("custom".into()), Type::Custom("custom".into()))
.category(Category::Custom("custom".into()))
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
_call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
todo!()
}
}
#[derive(Clone)]
pub struct IfMocked;
impl Command for IfMocked {
fn name(&self) -> &str {
"if"
}
fn usage(&self) -> &str {
"Mock if command."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build("if")
.required("cond", SyntaxShape::MathExpression, "condition to check")
.required(
"then_block",
SyntaxShape::Block,
"block to run if check succeeds",
)
.optional(
"else_expression",
Restrict closure expression to be something like `{|| ...}` (#8290) # Description As title, closes: #7921 closes: #8273 # User-Facing Changes when define a closure without pipe, nushell will raise error for now: ``` ❯ let x = {ss ss} Error: nu::parser::closure_missing_pipe × Missing || inside closure ╭─[entry #2:1:1] 1 │ let x = {ss ss} · ───┬─── · ╰── Parsing as a closure, but || is missing ╰──── help: Try add || to the beginning of closure ``` `any`, `each`, `all`, `where` command accepts closure, it forces user input closure like `{||`, or parse error will returned. ``` ❯ {major:2, minor:1, patch:4} | values | each { into string } Error: nu::parser::closure_missing_pipe × Missing || inside closure ╭─[entry #4:1:1] 1 │ {major:2, minor:1, patch:4} | values | each { into string } · ───────┬─────── · ╰── Parsing as a closure, but || is missing ╰──── help: Try add || to the beginning of closure ``` `with-env`, `do`, `def`, `try` are special, they still remain the same, although it says that it accepts a closure, but they don't need to be written like `{||`, it's more likely a block but can capture variable outside of scope: ``` ❯ def test [input] { echo [0 1 2] | do { do { echo $input } } }; test aaa aaa ``` Just realize that It's a big breaking change, we need to update config and scripts... # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-17 12:36:28 +00:00
SyntaxShape::Keyword(
b"else".to_vec(),
Box::new(SyntaxShape::OneOf(vec![
SyntaxShape::Block,
SyntaxShape::Expression,
])),
),
"expression or block to run if check fails",
)
.category(Category::Core)
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
_call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
todo!()
}
}
fn add_declarations(engine_state: &mut EngineState) {
let delta = {
let mut working_set = StateWorkingSet::new(engine_state);
working_set.add_decl(Box::new(Let));
working_set.add_decl(Box::new(Def));
working_set.add_decl(Box::new(AggCustom));
working_set.add_decl(Box::new(GroupByCustom));
working_set.add_decl(Box::new(GroupBy));
working_set.add_decl(Box::new(LsTest));
working_set.add_decl(Box::new(ToCustom));
working_set.add_decl(Box::new(AggMin));
working_set.add_decl(Box::new(Collect));
working_set.add_decl(Box::new(WithColumn));
working_set.add_decl(Box::new(IfMocked));
working_set.render()
};
engine_state
.merge_delta(delta)
.expect("Error merging delta");
}
#[test]
fn call_non_custom_types_test() {
let mut engine_state = EngineState::new();
add_declarations(&mut engine_state);
let mut working_set = StateWorkingSet::new(&engine_state);
let input = r#"ls | group-by name"#;
let block = parse(&mut working_set, None, input.as_bytes(), true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 2);
assert!(pipeline.elements[0].redirection.is_none());
assert!(pipeline.elements[1].redirection.is_none());
match &pipeline.elements[0].expr.expr {
Expr::Call(call) => {
Remove broken compile-time overload system (#9677) # Description This PR removes the compile-time overload system. Unfortunately, this system never worked correctly because in a gradual type system where types can be `Any`, you don't have enough information to correctly resolve function calls with overloads. These resolutions must be done at runtime, if they're supported. That said, there's a bit of work that needs to go into resolving input/output types (here overloads do not execute separate commands, but the same command and each overload explains how each output type corresponds to input types). This PR also removes the type scope, which would give incorrect answers in cases where multiple subexpressions were used in a pipeline. # User-Facing Changes Finishes removing compile-time overloads. These were only used in a few places in the code base, but it's possible it may impact user code. I'll mark this as breaking change so we can review. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect -A clippy::result_large_err` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- -c "use std testing; testing run-tests --path crates/nu-std"` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. -->
2023-07-13 19:05:03 +00:00
let expected_id = working_set.find_decl(b"ls").unwrap();
assert_eq!(call.decl_id, expected_id)
}
_ => panic!("Expected expression Call not found"),
}
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
match &pipeline.elements[1].expr.expr {
Expr::Call(call) => {
Remove broken compile-time overload system (#9677) # Description This PR removes the compile-time overload system. Unfortunately, this system never worked correctly because in a gradual type system where types can be `Any`, you don't have enough information to correctly resolve function calls with overloads. These resolutions must be done at runtime, if they're supported. That said, there's a bit of work that needs to go into resolving input/output types (here overloads do not execute separate commands, but the same command and each overload explains how each output type corresponds to input types). This PR also removes the type scope, which would give incorrect answers in cases where multiple subexpressions were used in a pipeline. # User-Facing Changes Finishes removing compile-time overloads. These were only used in a few places in the code base, but it's possible it may impact user code. I'll mark this as breaking change so we can review. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect -A clippy::result_large_err` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- -c "use std testing; testing run-tests --path crates/nu-std"` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. -->
2023-07-13 19:05:03 +00:00
let expected_id = working_set.find_decl(b"group-by").unwrap();
assert_eq!(call.decl_id, expected_id)
}
_ => panic!("Expected expression Call not found"),
}
}
#[test]
fn nested_operations_test() {
let mut engine_state = EngineState::new();
add_declarations(&mut engine_state);
let (block, delta) = {
let mut working_set = StateWorkingSet::new(&engine_state);
let input = r#"ls | to-custom | group-by name other | agg ("b" | min)"#;
let block = parse(&mut working_set, None, input.as_bytes(), true);
(block, working_set.render())
};
engine_state.merge_delta(delta).unwrap();
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert!(pipeline.elements[3].redirection.is_none());
match &pipeline.elements[3].expr.expr {
Expr::Call(call) => {
Span ID Refactor (Step 2): Make Call SpanId-friendly (#13268) <!-- if this PR closes one or more issues, you can automatically link the PR with them by using one of the [*linking keywords*](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword), e.g. - this PR should close #xxxx - fixes #xxxx you can also mention related issues, PRs or discussions! --> # Description <!-- Thank you for improving Nushell. Please, check our [contributing guide](../CONTRIBUTING.md) and talk to the core team before making major changes. Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience. --> Part of https://github.com/nushell/nushell/issues/12963, step 2. This PR refactors Call and related argument structures to remove their dependency on `Expression::span` which will be removed in the future. # User-Facing Changes <!-- List of all changes that impact the user experience here. This helps us keep track of breaking changes. --> Should be none. If you see some error messages that look broken, please report. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass (on Windows make sure to [enable developer mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging)) - `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. -->
2024-07-03 06:00:52 +00:00
let arg = &call.arguments.item[0];
match arg {
Argument::Positional(a) => match &a.expr {
Expr::FullCellPath(path) => match &path.head.expr {
Expr::Subexpression(id) => {
let block = engine_state.get_block(*id);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 2);
assert!(pipeline.elements[1].redirection.is_none());
match &pipeline.elements[1].expr.expr {
Expr::Call(call) => {
let working_set = StateWorkingSet::new(&engine_state);
Remove broken compile-time overload system (#9677) # Description This PR removes the compile-time overload system. Unfortunately, this system never worked correctly because in a gradual type system where types can be `Any`, you don't have enough information to correctly resolve function calls with overloads. These resolutions must be done at runtime, if they're supported. That said, there's a bit of work that needs to go into resolving input/output types (here overloads do not execute separate commands, but the same command and each overload explains how each output type corresponds to input types). This PR also removes the type scope, which would give incorrect answers in cases where multiple subexpressions were used in a pipeline. # User-Facing Changes Finishes removing compile-time overloads. These were only used in a few places in the code base, but it's possible it may impact user code. I'll mark this as breaking change so we can review. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect -A clippy::result_large_err` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- -c "use std testing; testing run-tests --path crates/nu-std"` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. -->
2023-07-13 19:05:03 +00:00
let expected_id = working_set.find_decl(b"min").unwrap();
assert_eq!(call.decl_id, expected_id)
}
_ => panic!("Expected expression Call not found"),
}
}
_ => panic!("Expected Subexpression not found"),
},
_ => panic!("Expected FullCellPath not found"),
},
_ => panic!("Expected Argument Positional not found"),
}
}
_ => panic!("Expected expression Call not found"),
}
}
#[test]
fn call_with_list_test() {
let mut engine_state = EngineState::new();
add_declarations(&mut engine_state);
let mut working_set = StateWorkingSet::new(&engine_state);
let input = r#"[[a b]; [1 2] [3 4]] | to-custom | with-column [ ("a" | min) ("b" | min) ] | collect"#;
let block = parse(&mut working_set, None, input.as_bytes(), true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
let pipeline = &block.pipelines[0];
assert!(pipeline.elements[2].redirection.is_none());
assert!(pipeline.elements[3].redirection.is_none());
match &pipeline.elements[2].expr.expr {
Expr::Call(call) => {
Remove broken compile-time overload system (#9677) # Description This PR removes the compile-time overload system. Unfortunately, this system never worked correctly because in a gradual type system where types can be `Any`, you don't have enough information to correctly resolve function calls with overloads. These resolutions must be done at runtime, if they're supported. That said, there's a bit of work that needs to go into resolving input/output types (here overloads do not execute separate commands, but the same command and each overload explains how each output type corresponds to input types). This PR also removes the type scope, which would give incorrect answers in cases where multiple subexpressions were used in a pipeline. # User-Facing Changes Finishes removing compile-time overloads. These were only used in a few places in the code base, but it's possible it may impact user code. I'll mark this as breaking change so we can review. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect -A clippy::result_large_err` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- -c "use std testing; testing run-tests --path crates/nu-std"` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. -->
2023-07-13 19:05:03 +00:00
let expected_id = working_set.find_decl(b"with-column").unwrap();
assert_eq!(call.decl_id, expected_id)
}
_ => panic!("Expected expression Call not found"),
}
IO and redirection overhaul (#11934) # Description The PR overhauls how IO redirection is handled, allowing more explicit and fine-grain control over `stdout` and `stderr` output as well as more efficient IO and piping. To summarize the changes in this PR: - Added a new `IoStream` type to indicate the intended destination for a pipeline element's `stdout` and `stderr`. - The `stdout` and `stderr` `IoStream`s are stored in the `Stack` and to avoid adding 6 additional arguments to every eval function and `Command::run`. The `stdout` and `stderr` streams can be temporarily overwritten through functions on `Stack` and these functions will return a guard that restores the original `stdout` and `stderr` when dropped. - In the AST, redirections are now directly part of a `PipelineElement` as a `Option<Redirection>` field instead of having multiple different `PipelineElement` enum variants for each kind of redirection. This required changes to the parser, mainly in `lite_parser.rs`. - `Command`s can also set a `IoStream` override/redirection which will apply to the previous command in the pipeline. This is used, for example, in `ignore` to allow the previous external command to have its stdout redirected to `Stdio::null()` at spawn time. In contrast, the current implementation has to create an os pipe and manually consume the output on nushell's side. File and pipe redirections (`o>`, `e>`, `e>|`, etc.) have precedence over overrides from commands. This PR improves piping and IO speed, partially addressing #10763. Using the `throughput` command from that issue, this PR gives the following speedup on my setup for the commands below: | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | --------------------------- | -------------:| ------------:| -----------:| | `throughput o> /dev/null` | 1169 | 52938 | 54305 | | `throughput \| ignore` | 840 | 55438 | N/A | | `throughput \| null` | Error | 53617 | N/A | | `throughput \| rg 'x'` | 1165 | 3049 | 3736 | | `(throughput) \| rg 'x'` | 810 | 3085 | 3815 | (Numbers above are the median samples for throughput) This PR also paves the way to refactor our `ExternalStream` handling in the various commands. For example, this PR already fixes the following code: ```nushell ^sh -c 'echo -n "hello "; sleep 0; echo "world"' | find "hello world" ``` This returns an empty list on 0.90.1 and returns a highlighted "hello world" on this PR. Since the `stdout` and `stderr` `IoStream`s are available to commands when they are run, then this unlocks the potential for more convenient behavior. E.g., the `find` command can disable its ansi highlighting if it detects that the output `IoStream` is not the terminal. Knowing the output streams will also allow background job output to be redirected more easily and efficiently. # User-Facing Changes - External commands returned from closures will be collected (in most cases): ```nushell 1..2 | each {|_| nu -c "print a" } ``` This gives `["a", "a"]` on this PR, whereas this used to print "a\na\n" and then return an empty list. ```nushell 1..2 | each {|_| nu -c "print -e a" } ``` This gives `["", ""]` and prints "a\na\n" to stderr, whereas this used to return an empty list and print "a\na\n" to stderr. - Trailing new lines are always trimmed for external commands when piping into internal commands or collecting it as a value. (Failure to decode the output as utf-8 will keep the trailing newline for the last binary value.) In the current nushell version, the following three code snippets differ only in parenthesis placement, but they all also have different outputs: 1. `1..2 | each { ^echo a }` ``` a a ╭────────────╮ │ empty list │ ╰────────────╯ ``` 2. `1..2 | each { (^echo a) }` ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` 3. `1..2 | (each { ^echo a })` ``` ╭───┬───╮ │ 0 │ a │ │ │ │ │ 1 │ a │ │ │ │ ╰───┴───╯ ``` But in this PR, the above snippets will all have the same output: ``` ╭───┬───╮ │ 0 │ a │ │ 1 │ a │ ╰───┴───╯ ``` - All existing flags on `run-external` are now deprecated. - File redirections now apply to all commands inside a code block: ```nushell (nu -c "print -e a"; nu -c "print -e b") e> test.out ``` This gives "a\nb\n" in `test.out` and prints nothing. The same result would happen when printing to stdout and using a `o>` file redirection. - External command output will (almost) never be ignored, and ignoring output must be explicit now: ```nushell (^echo a; ^echo b) ``` This prints "a\nb\n", whereas this used to print only "b\n". This only applies to external commands; values and internal commands not in return position will not print anything (e.g., `(echo a; echo b)` still only prints "b"). - `complete` now always captures stderr (`do` is not necessary). # After Submitting The language guide and other documentation will need to be updated.
2024-03-14 20:51:55 +00:00
match &pipeline.elements[3].expr.expr {
Expr::Call(call) => {
Remove broken compile-time overload system (#9677) # Description This PR removes the compile-time overload system. Unfortunately, this system never worked correctly because in a gradual type system where types can be `Any`, you don't have enough information to correctly resolve function calls with overloads. These resolutions must be done at runtime, if they're supported. That said, there's a bit of work that needs to go into resolving input/output types (here overloads do not execute separate commands, but the same command and each overload explains how each output type corresponds to input types). This PR also removes the type scope, which would give incorrect answers in cases where multiple subexpressions were used in a pipeline. # User-Facing Changes Finishes removing compile-time overloads. These were only used in a few places in the code base, but it's possible it may impact user code. I'll mark this as breaking change so we can review. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect -A clippy::result_large_err` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- -c "use std testing; testing run-tests --path crates/nu-std"` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. -->
2023-07-13 19:05:03 +00:00
let expected_id = working_set.find_decl(b"collect").unwrap();
assert_eq!(call.decl_id, expected_id)
}
_ => panic!("Expected expression Call not found"),
}
}
#[test]
fn operations_within_blocks_test() {
let mut engine_state = EngineState::new();
add_declarations(&mut engine_state);
let mut working_set = StateWorkingSet::new(&engine_state);
let inputs = vec![
Better errors when bash-like operators are used (#7241) # Description Adds improved errors for when a user uses a bashism that nu doesn't support. fixes #7237 Examples: ``` Error: nu::parser::shell_andand (link) × The '&&' operator is not supported in Nushell ╭─[entry #1:1:1] 1 │ ls && ls · ─┬ · ╰── instead of '&&', use ';' or 'and' ╰──── help: use ';' instead of the shell '&&', or 'and' instead of the boolean '&&' ``` ``` Error: nu::parser::shell_oror (link) × The '||' operator is not supported in Nushell ╭─[entry #8:1:1] 1 │ ls || ls · ─┬ · ╰── instead of '||', use 'try' or 'or' ╰──── help: use 'try' instead of the shell '||', or 'or' instead of the boolean '||' ``` ``` Error: nu::parser::shell_err (link) × The '2>' shell operation is 'err>' in Nushell. ╭─[entry #9:1:1] 1 │ foo 2> bar.txt · ─┬ · ╰── use 'err>' instead of '2>' in Nushell ╰──── ``` ``` Error: nu::parser::shell_outerr (link) × The '2>&1' shell operation is 'out+err>' in Nushell. ╭─[entry #10:1:1] 1 │ foo 2>&1 bar.txt · ──┬─ · ╰── use 'out+err>' instead of '2>&1' in Nushell ╰──── help: Nushell redirection will write all of stdout before stderr. ``` # User-Facing Changes **BREAKING CHANGES** This removes the `&&` and `||` operators. We previously supported by `&&`/`and` and `||`/`or`. With this change, only `and` and `or` are valid boolean operators. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-12-07 23:02:11 +00:00
r#"let a = 'b'; ($a == 'b') or ($a == 'b')"#,
r#"let a = 'b'; ($a == 'b') or ($a == 'b') and ($a == 'b')"#,
r#"let a = 1; ($a == 1) or ($a == 2) and ($a == 3)"#,
r#"let a = 'b'; if ($a == 'b') or ($a == 'b') { true } else { false }"#,
r#"let a = 1; if ($a == 1) or ($a > 0) { true } else { false }"#,
];
for input in inputs {
let block = parse(&mut working_set, None, input.as_bytes(), true);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 2, "testing: {input}");
}
}
#[test]
fn else_errors_correctly() {
let mut engine_state = EngineState::new();
add_declarations(&mut engine_state);
let mut working_set = StateWorkingSet::new(&engine_state);
parse(
&mut working_set,
None,
b"if false { 'a' } else { $foo }",
true,
);
assert!(matches!(
working_set.parse_errors.first(),
Some(ParseError::VariableNotFound(_, _))
));
}
#[test]
fn else_if_errors_correctly() {
let mut engine_state = EngineState::new();
add_declarations(&mut engine_state);
let mut working_set = StateWorkingSet::new(&engine_state);
parse(
&mut working_set,
None,
b"if false { 'a' } else $foo { 'b' }",
true,
);
assert!(matches!(
working_set.parse_errors.first(),
Some(ParseError::VariableNotFound(_, _))
));
}
#[rstest]
#[case::input_output(b"def q []: int -> int {1}", false)]
#[case::input_output(b"def q []: string -> string {'qwe'}", false)]
#[case::input_output(b"def q []: nothing -> nothing {null}", false)]
#[case::input_output(b"def q []: list<string> -> list<string> {[]}", false)]
#[case::input_output(
b"def q []: record<a: int b: int> -> record<c: int e: int> {{c: 1 e: 1}}",
false
)]
#[case::input_output(
b"def q []: table<a: int b: int> -> table<c: int e: int> {[{c: 1 e: 1}]}",
false
)]
#[case::input_output(
b"def q []: nothing -> record<c: record<a: int b: int> e: int> {{c: {a: 1 b: 2} e: 1}}",
false
)]
#[case::input_output(b"def q []: nothing -> list<string {[]}", true)]
#[case::input_output(b"def q []: nothing -> record<c: int e: int {{c: 1 e: 1}}", true)]
#[case::input_output(b"def q []: record<c: int e: int -> record<a: int> {{a: 1}}", true)]
#[case::input_output(b"def q []: nothing -> record<a: record<a: int> {{a: {a: 1}}}", true)]
#[case::vardecl(b"let a: int = 1", false)]
#[case::vardecl(b"let a: string = 'qwe'", false)]
#[case::vardecl(b"let a: nothing = null", false)]
#[case::vardecl(b"let a: list<string> = []", false)]
#[case::vardecl(b"let a: record<a: int b: int> = {a: 1 b: 1}", false)]
#[case::vardecl(
b"let a: record<c: record<a: int b: int> e: int> = {c: {a: 1 b: 2} e: 1}",
false
)]
#[case::vardecl(b"let a: table<a: int b: int> = [[a b]; [1 1]]", false)]
#[case::vardecl(b"let a: list<string asd> = []", true)]
#[case::vardecl(b"let a: record<a: int b: record<a: int> = {a: 1 b: {a: 1}}", true)]
fn test_type_annotations(#[case] phrase: &[u8], #[case] expect_errors: bool) {
let mut engine_state = EngineState::new();
add_declarations(&mut engine_state);
let mut working_set = StateWorkingSet::new(&engine_state);
// this should not panic
let _block = parse(&mut working_set, None, phrase, false);
// check that no parse errors happened
assert_eq!(
!working_set.parse_errors.is_empty(),
expect_errors,
"Got errors {:?}",
working_set.parse_errors
)
}
}
Allow string to copmpare with another string (#11590) # Description Nushell parser now reject comparison operator with 2 strings (e.g. `"abc" < "cba"`). This pr fixes it. ## before ```nu ~ ❯ "abc" < "bca" Error: nu::parser::unsupported_operation × less-than comparison is not supported on values of type string ╭─[entry #43:1:1] 1 │ "abc" < "bca" · ──┬── ┬ · │ ╰── doesn't support this value · ╰── string ╰──── ~ ❯ def foo []: nothing -> string { "abc" } ~ ❯ (foo) < "bca" Error: nu::parser::unsupported_operation × less-than comparison is not supported on values of type string ╭─[entry #53:1:1] 1 │ (foo) < "bca" · ──┬── ┬ · │ ╰── doesn't support this value · ╰── string ╰──── ``` ## after ```nu ~ ❯ "abc" < "bca" true ~ ❯ def foo []: nothing -> string { "abc" } ~ ❯ (foo) < "bca" true ``` # User-Facing Changes Following pattern will be allowed. | operator | type of lhs | type of rhs | result | | -------- | ----------- | ----------- | ------ | | `<` | string | string | bool | | `<=` | string | string | bool | | `>` | string | string | bool | | `>=` | string | string | bool | # Tests + Formatting - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass (on Windows make sure to [enable developer mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging)) - [x] `cargo run -- -c "use std testing; testing run-tests --path crates/nu-std"` to run the tests for the standard library # After Submitting
2024-01-21 13:43:40 +00:00
#[cfg(test)]
mod operator {
use super::*;
#[rstest]
#[case(br#""abc" < "bca""#, "string < string")]
#[case(br#""abc" <= "bca""#, "string <= string")]
#[case(br#""abc" > "bca""#, "string > string")]
#[case(br#""abc" >= "bca""#, "string >= string")]
fn parse_comparison_operators_with_string_and_string(
#[case] expr: &[u8],
#[case] test_tag: &str,
) {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
parse(&mut working_set, None, expr, false);
assert_eq!(
working_set.parse_errors.len(),
0,
"{test_tag}: expected to be parsed successfully, but failed."
);
}
}
Span ID Refactor (Step 2): Make Call SpanId-friendly (#13268) <!-- if this PR closes one or more issues, you can automatically link the PR with them by using one of the [*linking keywords*](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword), e.g. - this PR should close #xxxx - fixes #xxxx you can also mention related issues, PRs or discussions! --> # Description <!-- Thank you for improving Nushell. Please, check our [contributing guide](../CONTRIBUTING.md) and talk to the core team before making major changes. Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience. --> Part of https://github.com/nushell/nushell/issues/12963, step 2. This PR refactors Call and related argument structures to remove their dependency on `Expression::span` which will be removed in the future. # User-Facing Changes <!-- List of all changes that impact the user experience here. This helps us keep track of breaking changes. --> Should be none. If you see some error messages that look broken, please report. # Tests + Formatting <!-- Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass (on Windows make sure to [enable developer mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging)) - `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` --> # After Submitting <!-- If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. -->
2024-07-03 06:00:52 +00:00
#[cfg(test)]
mod test {
use nu_parser::named_arg_span;
use nu_protocol::ast::{Argument, Call, Expression};
use nu_protocol::engine::{EngineState, StateWorkingSet};
use nu_protocol::{Span, Spanned};
#[test]
fn argument_span_named() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let named_span = Span::new(2, 3);
let short_span = Span::new(5, 7);
let expr_span = Span::new(11, 13);
let _ = working_set.add_span(named_span);
let _ = working_set.add_span(short_span);
let _ = working_set.add_span(expr_span);
let named = Spanned {
item: "named".to_string(),
span: named_span,
};
let short = Spanned {
item: "short".to_string(),
span: short_span,
};
let expr = Expression::garbage(&mut working_set, expr_span);
let arg_span = named_arg_span(&working_set, &named, &None, &None);
assert_eq!(Span::new(2, 3), arg_span);
let arg_span = named_arg_span(&working_set, &named, &Some(short.clone()), &None);
assert_eq!(Span::new(2, 7), arg_span);
let arg_span = named_arg_span(&working_set, &named, &None, &Some(expr.clone()));
assert_eq!(Span::new(2, 13), arg_span);
let arg_span = named_arg_span(
&working_set,
&named,
&Some(short.clone()),
&Some(expr.clone()),
);
assert_eq!(Span::new(2, 13), arg_span);
}
#[test]
fn argument_span_positional() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let span = Span::new(2, 3);
let _ = working_set.add_span(span);
let expr = Expression::garbage(&mut working_set, span);
let arg = Argument::Positional(expr);
assert_eq!(span, arg.span(&working_set));
}
#[test]
fn argument_span_unknown() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let span = Span::new(2, 3);
let _ = working_set.add_span(span);
let expr = Expression::garbage(&mut working_set, span);
let arg = Argument::Unknown(expr);
assert_eq!(span, arg.span(&working_set));
}
#[test]
fn call_arguments_span() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let arg1_span = Span::new(2, 3);
let arg2_span = Span::new(5, 7);
let mut call = Call::new(Span::new(0, 1), Span::concat(&[arg1_span, arg2_span]));
call.add_positional(Expression::garbage(&mut working_set, arg1_span));
call.add_positional(Expression::garbage(&mut working_set, arg2_span));
assert_eq!(Span::new(2, 7), call.arguments_span());
}
}