From 6454bf69aa239a09777f04c9c42fb5e16d6eb1c9 Mon Sep 17 00:00:00 2001 From: JT <547158+jntrnr@users.noreply.github.com> Date: Sat, 19 Nov 2022 10:46:48 +1300 Subject: [PATCH] Parser refactoring for improving pipelines (#7162) * Remove lite_parse file * Add LiteElement to represent different pipeline elem types * Add PipelineElement to Pipelines * Remove lite_parse specific tests --- crates/nu-cli/src/completions/completer.rs | 437 +++++++------ crates/nu-cli/src/syntax_highlight.rs | 29 +- crates/nu-command/src/example_test.rs | 6 +- crates/nu-command/src/formats/from/nuon.rs | 25 +- crates/nu-command/tests/main.rs | 5 +- crates/nu-engine/src/eval.rs | 56 +- crates/nu-parser/src/flatten.rs | 20 +- crates/nu-parser/src/lib.rs | 11 +- crates/nu-parser/src/lite_parse.rs | 184 ------ crates/nu-parser/src/parse_keywords.rs | 209 +++--- crates/nu-parser/src/parser.rs | 707 +++++++++++++++------ crates/nu-parser/tests/test_lite_parser.rs | 242 ------- crates/nu-parser/tests/test_parser.rs | 215 ++++--- crates/nu-protocol/src/ast/expression.rs | 64 +- crates/nu-protocol/src/ast/pipeline.rs | 79 ++- src/main.rs | 6 +- 16 files changed, 1196 insertions(+), 1099 deletions(-) delete mode 100644 crates/nu-parser/src/lite_parse.rs delete mode 100644 crates/nu-parser/tests/test_lite_parser.rs diff --git a/crates/nu-cli/src/completions/completer.rs b/crates/nu-cli/src/completions/completer.rs index 2c82bf6f81..5acd89fefb 100644 --- a/crates/nu-cli/src/completions/completer.rs +++ b/crates/nu-cli/src/completions/completer.rs @@ -5,6 +5,7 @@ use crate::completions::{ use nu_engine::eval_block; use nu_parser::{flatten_expression, parse, FlatShape}; use nu_protocol::{ + ast::PipelineElement, engine::{EngineState, Stack, StateWorkingSet}, BlockId, PipelineData, Span, Value, }; @@ -131,139 +132,62 @@ impl NuCompleter { let (output, _err) = parse(&mut working_set, Some("completer"), &new_line, false, &[]); for pipeline in output.pipelines.into_iter() { - for expr in pipeline.expressions { - let flattened: Vec<_> = flatten_expression(&working_set, &expr); - let span_offset: usize = alias_offset.iter().sum(); - let mut spans: Vec = vec![]; + for pipeline_element in pipeline.elements { + match pipeline_element { + PipelineElement::Expression(expr) + | PipelineElement::Redirect(expr) + | PipelineElement::And(expr) + | PipelineElement::Or(expr) => { + let flattened: Vec<_> = flatten_expression(&working_set, &expr); + let span_offset: usize = alias_offset.iter().sum(); + let mut spans: Vec = vec![]; - for (flat_idx, flat) in flattened.iter().enumerate() { - // Read the current spam to string - let current_span = working_set.get_span_contents(flat.0).to_vec(); - let current_span_str = String::from_utf8_lossy(¤t_span); + for (flat_idx, flat) in flattened.iter().enumerate() { + // Read the current spam to string + let current_span = working_set.get_span_contents(flat.0).to_vec(); + let current_span_str = String::from_utf8_lossy(¤t_span); - // Skip the last 'a' as span item - if flat_idx == flattened.len() - 1 { - let mut chars = current_span_str.chars(); - chars.next_back(); - let current_span_str = chars.as_str().to_owned(); - spans.push(current_span_str.to_string()); - } else { - spans.push(current_span_str.to_string()); - } - - // Complete based on the last span - if pos + span_offset >= flat.0.start && pos + span_offset < flat.0.end { - // Context variables - let most_left_var = - most_left_variable(flat_idx, &working_set, flattened.clone()); - - // Create a new span - let new_span = if flat_idx == 0 { - Span { - start: flat.0.start, - end: flat.0.end - 1 - span_offset, - } - } else { - Span { - start: flat.0.start - span_offset, - end: flat.0.end - 1 - span_offset, - } - }; - - // Parses the prefix. Completion should look up to the cursor position, not after. - let mut prefix = working_set.get_span_contents(flat.0).to_vec(); - let index = pos - (flat.0.start - span_offset); - prefix.drain(index..); - - // Variables completion - if prefix.starts_with(b"$") || most_left_var.is_some() { - let mut completer = VariableCompletion::new( - self.engine_state.clone(), - self.stack.clone(), - most_left_var.unwrap_or((vec![], vec![])), - ); - - return self.process_completion( - &mut completer, - &working_set, - prefix, - new_span, - offset, - pos, - ); - } - - // Flags completion - if prefix.starts_with(b"-") { - // Try to complete flag internally - let mut completer = FlagCompletion::new(expr.clone()); - let result = self.process_completion( - &mut completer, - &working_set, - prefix.clone(), - new_span, - offset, - pos, - ); - - if !result.is_empty() { - return result; + // Skip the last 'a' as span item + if flat_idx == flattened.len() - 1 { + let mut chars = current_span_str.chars(); + chars.next_back(); + let current_span_str = chars.as_str().to_owned(); + spans.push(current_span_str.to_string()); + } else { + spans.push(current_span_str.to_string()); } - // We got no results for internal completion - // now we can check if external completer is set and use it - if let Some(block_id) = config.external_completer { - if let Some(external_result) = - self.external_completion(block_id, &spans, offset, new_span) - { - return external_result; - } - } - } + // Complete based on the last span + if pos + span_offset >= flat.0.start && pos + span_offset < flat.0.end { + // Context variables + let most_left_var = + most_left_variable(flat_idx, &working_set, flattened.clone()); - // specially check if it is currently empty - always complete commands - if flat_idx == 0 && working_set.get_span_contents(new_span).is_empty() { - let mut completer = CommandCompletion::new( - self.engine_state.clone(), - &working_set, - flattened.clone(), - // flat_idx, - FlatShape::String, - true, - ); - return self.process_completion( - &mut completer, - &working_set, - prefix, - new_span, - offset, - pos, - ); - } + // Create a new span + let new_span = if flat_idx == 0 { + Span { + start: flat.0.start, + end: flat.0.end - 1 - span_offset, + } + } else { + Span { + start: flat.0.start - span_offset, + end: flat.0.end - 1 - span_offset, + } + }; - // Completions that depends on the previous expression (e.g: use, source-env) - if flat_idx > 0 { - if let Some(previous_expr) = flattened.get(flat_idx - 1) { - // Read the content for the previous expression - let prev_expr_str = - working_set.get_span_contents(previous_expr.0).to_vec(); + // Parses the prefix. Completion should look up to the cursor position, not after. + let mut prefix = working_set.get_span_contents(flat.0).to_vec(); + let index = pos - (flat.0.start - span_offset); + prefix.drain(index..); - // Completion for .nu files - if prev_expr_str == b"use" || prev_expr_str == b"source-env" { - let mut completer = - DotNuCompletion::new(self.engine_state.clone()); - - return self.process_completion( - &mut completer, - &working_set, - prefix, - new_span, - offset, - pos, + // Variables completion + if prefix.starts_with(b"$") || most_left_var.is_some() { + let mut completer = VariableCompletion::new( + self.engine_state.clone(), + self.stack.clone(), + most_left_var.unwrap_or((vec![], vec![])), ); - } else if prev_expr_str == b"ls" { - let mut completer = - FileCompletion::new(self.engine_state.clone()); return self.process_completion( &mut completer, @@ -274,101 +198,190 @@ impl NuCompleter { pos, ); } - } - } - // Match other types - match &flat.1 { - FlatShape::Custom(decl_id) => { - let mut completer = CustomCompletion::new( - self.engine_state.clone(), - self.stack.clone(), - *decl_id, - initial_line, - ); + // Flags completion + if prefix.starts_with(b"-") { + // Try to complete flag internally + let mut completer = FlagCompletion::new(expr.clone()); + let result = self.process_completion( + &mut completer, + &working_set, + prefix.clone(), + new_span, + offset, + pos, + ); - return self.process_completion( - &mut completer, - &working_set, - prefix, - new_span, - offset, - pos, - ); - } - FlatShape::Directory => { - let mut completer = - DirectoryCompletion::new(self.engine_state.clone()); + if !result.is_empty() { + return result; + } - return self.process_completion( - &mut completer, - &working_set, - prefix, - new_span, - offset, - pos, - ); - } - FlatShape::Filepath | FlatShape::GlobPattern => { - let mut completer = FileCompletion::new(self.engine_state.clone()); - - return self.process_completion( - &mut completer, - &working_set, - prefix, - new_span, - offset, - pos, - ); - } - flat_shape => { - let mut completer = CommandCompletion::new( - self.engine_state.clone(), - &working_set, - flattened.clone(), - // flat_idx, - flat_shape.clone(), - false, - ); - - let mut out: Vec<_> = self.process_completion( - &mut completer, - &working_set, - prefix.clone(), - new_span, - offset, - pos, - ); - - if !out.is_empty() { - return out; - } - - // Try to complete using an external completer (if set) - if let Some(block_id) = config.external_completer { - if let Some(external_result) = - self.external_completion(block_id, &spans, offset, new_span) - { - return external_result; + // We got no results for internal completion + // now we can check if external completer is set and use it + if let Some(block_id) = config.external_completer { + if let Some(external_result) = self + .external_completion(block_id, &spans, offset, new_span) + { + return external_result; + } } } - // Check for file completion - let mut completer = FileCompletion::new(self.engine_state.clone()); - out = self.process_completion( - &mut completer, - &working_set, - prefix, - new_span, - offset, - pos, - ); - - if !out.is_empty() { - return out; + // specially check if it is currently empty - always complete commands + if flat_idx == 0 + && working_set.get_span_contents(new_span).is_empty() + { + let mut completer = CommandCompletion::new( + self.engine_state.clone(), + &working_set, + flattened.clone(), + // flat_idx, + FlatShape::String, + true, + ); + return self.process_completion( + &mut completer, + &working_set, + prefix, + new_span, + offset, + pos, + ); } + + // Completions that depends on the previous expression (e.g: use, source-env) + if flat_idx > 0 { + if let Some(previous_expr) = flattened.get(flat_idx - 1) { + // Read the content for the previous expression + let prev_expr_str = + working_set.get_span_contents(previous_expr.0).to_vec(); + + // Completion for .nu files + if prev_expr_str == b"use" || prev_expr_str == b"source-env" + { + let mut completer = + DotNuCompletion::new(self.engine_state.clone()); + + return self.process_completion( + &mut completer, + &working_set, + prefix, + new_span, + offset, + pos, + ); + } else if prev_expr_str == b"ls" { + let mut completer = + FileCompletion::new(self.engine_state.clone()); + + return self.process_completion( + &mut completer, + &working_set, + prefix, + new_span, + offset, + pos, + ); + } + } + } + + // Match other types + match &flat.1 { + FlatShape::Custom(decl_id) => { + let mut completer = CustomCompletion::new( + self.engine_state.clone(), + self.stack.clone(), + *decl_id, + initial_line, + ); + + return self.process_completion( + &mut completer, + &working_set, + prefix, + new_span, + offset, + pos, + ); + } + FlatShape::Directory => { + let mut completer = + DirectoryCompletion::new(self.engine_state.clone()); + + return self.process_completion( + &mut completer, + &working_set, + prefix, + new_span, + offset, + pos, + ); + } + FlatShape::Filepath | FlatShape::GlobPattern => { + let mut completer = + FileCompletion::new(self.engine_state.clone()); + + return self.process_completion( + &mut completer, + &working_set, + prefix, + new_span, + offset, + pos, + ); + } + flat_shape => { + let mut completer = CommandCompletion::new( + self.engine_state.clone(), + &working_set, + flattened.clone(), + // flat_idx, + flat_shape.clone(), + false, + ); + + let mut out: Vec<_> = self.process_completion( + &mut completer, + &working_set, + prefix.clone(), + new_span, + offset, + pos, + ); + + if !out.is_empty() { + return out; + } + + // Try to complete using an external completer (if set) + if let Some(block_id) = config.external_completer { + if let Some(external_result) = self.external_completion( + block_id, &spans, offset, new_span, + ) { + return external_result; + } + } + + // Check for file completion + let mut completer = + FileCompletion::new(self.engine_state.clone()); + out = self.process_completion( + &mut completer, + &working_set, + prefix, + new_span, + offset, + pos, + ); + + if !out.is_empty() { + return out; + } + } + }; } - }; + } } } } diff --git a/crates/nu-cli/src/syntax_highlight.rs b/crates/nu-cli/src/syntax_highlight.rs index fc57247971..bce804ddb2 100644 --- a/crates/nu-cli/src/syntax_highlight.rs +++ b/crates/nu-cli/src/syntax_highlight.rs @@ -2,7 +2,7 @@ use log::trace; use nu_ansi_term::Style; use nu_color_config::{get_matching_brackets_style, get_shape_color}; use nu_parser::{flatten_block, parse, FlatShape}; -use nu_protocol::ast::{Argument, Block, Expr, Expression}; +use nu_protocol::ast::{Argument, Block, Expr, Expression, PipelineElement}; use nu_protocol::engine::{EngineState, StateWorkingSet}; use nu_protocol::{Config, Span}; use reedline::{Highlighter, StyledText}; @@ -230,16 +230,23 @@ fn find_matching_block_end_in_block( global_cursor_offset: usize, ) -> Option { for p in &block.pipelines { - for e in &p.expressions { - if e.span.contains(global_cursor_offset) { - if let Some(pos) = find_matching_block_end_in_expr( - line, - working_set, - e, - global_span_offset, - global_cursor_offset, - ) { - return Some(pos); + for e in &p.elements { + match e { + PipelineElement::Expression(e) + | PipelineElement::Redirect(e) + | PipelineElement::And(e) + | PipelineElement::Or(e) => { + if e.span.contains(global_cursor_offset) { + if let Some(pos) = find_matching_block_end_in_expr( + line, + working_set, + e, + global_span_offset, + global_cursor_offset, + ) { + return Some(pos); + } + } } } } diff --git a/crates/nu-command/src/example_test.rs b/crates/nu-command/src/example_test.rs index 20eae2f741..70bf8fb01c 100644 --- a/crates/nu-command/src/example_test.rs +++ b/crates/nu-command/src/example_test.rs @@ -310,10 +310,10 @@ mod test_examples { let (mut block, delta) = parse(src, engine_state); match block.pipelines.len() { 1 => { - let n_expressions = block.pipelines[0].expressions.len(); - block.pipelines[0].expressions.truncate(&n_expressions - 1); + let n_expressions = block.pipelines[0].elements.len(); + block.pipelines[0].elements.truncate(&n_expressions - 1); - if !block.pipelines[0].expressions.is_empty() { + if !block.pipelines[0].elements.is_empty() { let empty_input = PipelineData::new(Span::test_data()); Some(eval_block(block, empty_input, cwd, engine_state, delta)) } else { diff --git a/crates/nu-command/src/formats/from/nuon.rs b/crates/nu-command/src/formats/from/nuon.rs index 6657480d4a..8a2cd70434 100644 --- a/crates/nu-command/src/formats/from/nuon.rs +++ b/crates/nu-command/src/formats/from/nuon.rs @@ -1,4 +1,4 @@ -use nu_protocol::ast::{Call, Expr, Expression}; +use nu_protocol::ast::{Call, Expr, Expression, PipelineElement}; use nu_protocol::engine::{Command, EngineState, Stack, StateWorkingSet}; use nu_protocol::{ Category, Example, IntoPipelineData, PipelineData, Range, ShellError, Signature, Span, Type, @@ -88,15 +88,11 @@ impl Command for FromNuon { let (lexed, err) = nu_parser::lex(string_input.as_bytes(), 0, &[b'\n', b'\r'], &[], true); error = error.or(err); - let (lite_block, err) = nu_parser::lite_parse(&lexed); - error = error.or(err); - - let (mut block, err) = - nu_parser::parse_block(&mut working_set, &lite_block, true, &[], false); + let (mut block, err) = nu_parser::parse_block(&mut working_set, &lexed, true, &[], false); error = error.or(err); if let Some(pipeline) = block.pipelines.get(1) { - if let Some(expr) = pipeline.expressions.get(0) { + if let Some(element) = pipeline.elements.get(0) { return Err(ShellError::GenericError( "error when loading nuon text".into(), "could not load nuon text".into(), @@ -106,7 +102,7 @@ impl Command for FromNuon { string_input, "error when loading".into(), "excess values when loading".into(), - expr.span, + element.span(), )], )); } else { @@ -136,7 +132,7 @@ impl Command for FromNuon { } else { let mut pipeline = block.pipelines.remove(0); - if let Some(expr) = pipeline.expressions.get(1) { + if let Some(expr) = pipeline.elements.get(1) { return Err(ShellError::GenericError( "error when loading nuon text".into(), "could not load nuon text".into(), @@ -146,12 +142,12 @@ impl Command for FromNuon { string_input, "error when loading".into(), "detected a pipeline in nuon file".into(), - expr.span, + expr.span(), )], )); } - if pipeline.expressions.is_empty() { + if pipeline.elements.is_empty() { Expression { expr: Expr::Nothing, span: head, @@ -159,7 +155,12 @@ impl Command for FromNuon { ty: Type::Nothing, } } else { - pipeline.expressions.remove(0) + match pipeline.elements.remove(0) { + PipelineElement::Expression(expression) + | PipelineElement::Redirect(expression) + | PipelineElement::And(expression) + | PipelineElement::Or(expression) => expression, + } } }; diff --git a/crates/nu-command/tests/main.rs b/crates/nu-command/tests/main.rs index 4eedfd2b64..125033a17e 100644 --- a/crates/nu-command/tests/main.rs +++ b/crates/nu-command/tests/main.rs @@ -10,15 +10,14 @@ mod format_conversions; #[quickcheck] fn quickcheck_parse(data: String) -> bool { let (tokens, err) = nu_parser::lex(data.as_bytes(), 0, b"", b"", true); - let (lite_block, err2) = nu_parser::lite_parse(&tokens); - if err.is_none() && err2.is_none() { + if err.is_none() { let context = create_default_context(); { let mut working_set = StateWorkingSet::new(&context); working_set.add_file("quickcheck".into(), data.as_bytes()); - let _ = nu_parser::parse_block(&mut working_set, &lite_block, false, &[], false); + let _ = nu_parser::parse_block(&mut working_set, &tokens, false, &[], false); } } true diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index 1a023e67d6..a57255e5ef 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -3,7 +3,7 @@ use nu_path::expand_path_with; use nu_protocol::{ ast::{ Assignment, Bits, Block, Boolean, Call, Comparison, Expr, Expression, Math, Operator, - PathMember, + PathMember, PipelineElement, }, engine::{EngineState, Stack}, Config, HistoryFileFormat, IntoInterruptiblePipelineData, IntoPipelineData, ListStream, @@ -786,6 +786,50 @@ fn might_consume_external_result(input: PipelineData) -> (PipelineData, bool) { } } +pub fn eval_element_with_input( + engine_state: &EngineState, + stack: &mut Stack, + element: &PipelineElement, + input: PipelineData, + redirect_stdout: bool, + redirect_stderr: bool, +) -> Result<(PipelineData, bool), ShellError> { + match element { + PipelineElement::Expression(expr) => eval_expression_with_input( + engine_state, + stack, + expr, + input, + redirect_stdout, + redirect_stderr, + ), + PipelineElement::Redirect(expr) => eval_expression_with_input( + engine_state, + stack, + expr, + input, + redirect_stdout, + redirect_stderr, + ), + PipelineElement::And(expr) => eval_expression_with_input( + engine_state, + stack, + expr, + input, + redirect_stdout, + redirect_stderr, + ), + PipelineElement::Or(expr) => eval_expression_with_input( + engine_state, + stack, + expr, + input, + redirect_stdout, + redirect_stderr, + ), + } +} + pub fn eval_block( engine_state: &EngineState, stack: &mut Stack, @@ -796,14 +840,14 @@ pub fn eval_block( ) -> Result { let num_pipelines = block.len(); for (pipeline_idx, pipeline) in block.pipelines.iter().enumerate() { - for (i, elem) in pipeline.expressions.iter().enumerate() { + for (i, elem) in pipeline.elements.iter().enumerate() { // if eval internal command failed, it can just make early return with `Err(ShellError)`. - let eval_result = eval_expression_with_input( + let eval_result = eval_element_with_input( engine_state, stack, elem, input, - redirect_stdout || (i != pipeline.expressions.len() - 1), + redirect_stdout || (i != pipeline.elements.len() - 1), redirect_stderr, )?; input = eval_result.0; @@ -901,8 +945,8 @@ pub fn eval_subexpression( mut input: PipelineData, ) -> Result { for pipeline in block.pipelines.iter() { - for expr in pipeline.expressions.iter() { - input = eval_expression_with_input(engine_state, stack, expr, input, true, false)?.0 + for expr in pipeline.elements.iter() { + input = eval_element_with_input(engine_state, stack, expr, input, true, false)?.0 } } diff --git a/crates/nu-parser/src/flatten.rs b/crates/nu-parser/src/flatten.rs index ba6b7e6534..e9a197aa16 100644 --- a/crates/nu-parser/src/flatten.rs +++ b/crates/nu-parser/src/flatten.rs @@ -1,4 +1,6 @@ -use nu_protocol::ast::{Block, Expr, Expression, ImportPatternMember, PathMember, Pipeline}; +use nu_protocol::ast::{ + Block, Expr, Expression, ImportPatternMember, PathMember, Pipeline, PipelineElement, +}; use nu_protocol::DeclId; use nu_protocol::{engine::StateWorkingSet, Span}; use std::fmt::{Display, Formatter, Result}; @@ -495,13 +497,25 @@ pub fn flatten_expression( } } +pub fn flatten_pipeline_element( + working_set: &StateWorkingSet, + pipeline_element: &PipelineElement, +) -> Vec<(Span, FlatShape)> { + match pipeline_element { + PipelineElement::Expression(expr) + | PipelineElement::Redirect(expr) + | PipelineElement::And(expr) + | PipelineElement::Or(expr) => flatten_expression(working_set, expr), + } +} + pub fn flatten_pipeline( working_set: &StateWorkingSet, pipeline: &Pipeline, ) -> Vec<(Span, FlatShape)> { let mut output = vec![]; - for expr in &pipeline.expressions { - output.extend(flatten_expression(working_set, expr)) + for expr in &pipeline.elements { + output.extend(flatten_pipeline_element(working_set, expr)) } output } diff --git a/crates/nu-parser/src/lib.rs b/crates/nu-parser/src/lib.rs index 00aa1813b4..66f56b70da 100644 --- a/crates/nu-parser/src/lib.rs +++ b/crates/nu-parser/src/lib.rs @@ -3,22 +3,23 @@ mod errors; mod flatten; mod known_external; mod lex; -mod lite_parse; mod parse_keywords; mod parser; mod type_check; pub use deparse::{escape_for_script_arg, escape_quote_string}; pub use errors::ParseError; -pub use flatten::{flatten_block, flatten_expression, flatten_pipeline, FlatShape}; +pub use flatten::{ + flatten_block, flatten_expression, flatten_pipeline, flatten_pipeline_element, FlatShape, +}; pub use known_external::KnownExternal; pub use lex::{lex, Token, TokenContents}; -pub use lite_parse::{lite_parse, LiteBlock}; pub use parse_keywords::*; pub use parser::{ - is_math_expression_like, parse, parse_block, parse_duration_bytes, parse_expression, - parse_external_call, trim_quotes, trim_quotes_str, unescape_unquote_string, Import, + is_math_expression_like, lite_parse, parse, parse_block, parse_duration_bytes, + parse_expression, parse_external_call, trim_quotes, trim_quotes_str, unescape_unquote_string, + Import, LiteBlock, LiteElement, }; #[cfg(feature = "plugin")] diff --git a/crates/nu-parser/src/lite_parse.rs b/crates/nu-parser/src/lite_parse.rs deleted file mode 100644 index 05d1800e6b..0000000000 --- a/crates/nu-parser/src/lite_parse.rs +++ /dev/null @@ -1,184 +0,0 @@ -use crate::{ParseError, Token, TokenContents}; -use nu_protocol::Span; - -#[derive(Debug)] -pub struct LiteCommand { - pub comments: Vec, - pub parts: Vec, -} - -impl Default for LiteCommand { - fn default() -> Self { - Self::new() - } -} - -impl LiteCommand { - pub fn new() -> Self { - Self { - comments: vec![], - parts: vec![], - } - } - - pub fn push(&mut self, span: Span) { - self.parts.push(span); - } - - pub fn is_empty(&self) -> bool { - self.parts.is_empty() - } -} - -#[derive(Debug)] -pub struct LitePipeline { - pub commands: Vec, -} - -impl Default for LitePipeline { - fn default() -> Self { - Self::new() - } -} - -impl LitePipeline { - pub fn new() -> Self { - Self { commands: vec![] } - } - - pub fn push(&mut self, command: LiteCommand) { - self.commands.push(command); - } - - pub fn is_empty(&self) -> bool { - self.commands.is_empty() - } -} - -#[derive(Debug)] -pub struct LiteBlock { - pub block: Vec, -} - -impl Default for LiteBlock { - fn default() -> Self { - Self::new() - } -} - -impl LiteBlock { - pub fn new() -> Self { - Self { block: vec![] } - } - - pub fn push(&mut self, pipeline: LitePipeline) { - self.block.push(pipeline); - } - - pub fn is_empty(&self) -> bool { - self.block.is_empty() - } -} - -pub fn lite_parse(tokens: &[Token]) -> (LiteBlock, Option) { - let mut block = LiteBlock::new(); - let mut curr_pipeline = LitePipeline::new(); - let mut curr_command = LiteCommand::new(); - - let mut last_token = TokenContents::Eol; - - let mut curr_comment: Option> = None; - - for token in tokens.iter() { - match &token.contents { - TokenContents::Item => { - // If we have a comment, go ahead and attach it - if let Some(curr_comment) = curr_comment.take() { - curr_command.comments = curr_comment; - } - curr_command.push(token.span); - last_token = TokenContents::Item; - } - TokenContents::Pipe => { - if !curr_command.is_empty() { - curr_pipeline.push(curr_command); - curr_command = LiteCommand::new(); - } - last_token = TokenContents::Pipe; - } - TokenContents::Eol => { - if last_token != TokenContents::Pipe { - if !curr_command.is_empty() { - curr_pipeline.push(curr_command); - - curr_command = LiteCommand::new(); - } - - if !curr_pipeline.is_empty() { - block.push(curr_pipeline); - - curr_pipeline = LitePipeline::new(); - } - } - - if last_token == TokenContents::Eol { - // Clear out the comment as we're entering a new comment - curr_comment = None; - } - - last_token = TokenContents::Eol; - } - TokenContents::Semicolon => { - if !curr_command.is_empty() { - curr_pipeline.push(curr_command); - - curr_command = LiteCommand::new(); - } - - if !curr_pipeline.is_empty() { - block.push(curr_pipeline); - - curr_pipeline = LitePipeline::new(); - } - - last_token = TokenContents::Semicolon; - } - TokenContents::Comment => { - // Comment is beside something - if last_token != TokenContents::Eol { - curr_command.comments.push(token.span); - curr_comment = None; - } else { - // Comment precedes something - if let Some(curr_comment) = &mut curr_comment { - curr_comment.push(token.span); - } else { - curr_comment = Some(vec![token.span]); - } - } - - last_token = TokenContents::Comment; - } - } - } - - if !curr_command.is_empty() { - curr_pipeline.push(curr_command); - } - - if !curr_pipeline.is_empty() { - block.push(curr_pipeline); - } - - if last_token == TokenContents::Pipe { - ( - block, - Some(ParseError::UnexpectedEof( - "pipeline missing end".into(), - tokens[tokens.len() - 1].span, - )), - ) - } else { - (block, None) - } -} diff --git a/crates/nu-parser/src/parse_keywords.rs b/crates/nu-parser/src/parse_keywords.rs index 65b6e4e8af..e1fc5d3424 100644 --- a/crates/nu-parser/src/parse_keywords.rs +++ b/crates/nu-parser/src/parse_keywords.rs @@ -2,7 +2,7 @@ use nu_path::canonicalize_with; use nu_protocol::{ ast::{ Argument, Block, Call, Expr, Expression, ImportPattern, ImportPatternHead, - ImportPatternMember, PathMember, Pipeline, + ImportPatternMember, PathMember, Pipeline, PipelineElement, }, engine::{StateWorkingSet, DEFAULT_OVERLAY_NAME}, span, BlockId, Exportable, Module, PositionalArg, Span, Spanned, SyntaxShape, Type, @@ -16,12 +16,11 @@ static PLUGIN_DIRS_ENV: &str = "NU_PLUGIN_DIRS"; use crate::{ known_external::KnownExternal, - lex, lite_parse, - lite_parse::LiteCommand, + lex, parser::{ - check_call, check_name, garbage, garbage_pipeline, parse, parse_internal_call, + check_call, check_name, garbage, garbage_pipeline, lite_parse, parse, parse_internal_call, parse_multispan_value, parse_signature, parse_string, parse_var_with_opt_type, trim_quotes, - ParsedInternalCall, + LiteCommand, LiteElement, ParsedInternalCall, }, unescape_unquote_string, ParseError, }; @@ -866,10 +865,10 @@ pub fn parse_export_in_module( }; // Trying to warp the 'def' call into the 'export def' in a very clumsy way - if let Some(Expression { + if let Some(PipelineElement::Expression(Expression { expr: Expr::Call(ref def_call), .. - }) = pipeline.expressions.get(0) + })) = pipeline.elements.get(0) { call = def_call.clone(); @@ -931,10 +930,10 @@ pub fn parse_export_in_module( }; // Trying to warp the 'def' call into the 'export def' in a very clumsy way - if let Some(Expression { + if let Some(PipelineElement::Expression(Expression { expr: Expr::Call(ref def_call), .. - }) = pipeline.expressions.get(0) + })) = pipeline.elements.get(0) { call = def_call.clone(); @@ -997,10 +996,10 @@ pub fn parse_export_in_module( }; // Trying to warp the 'def' call into the 'export def' in a very clumsy way - if let Some(Expression { + if let Some(PipelineElement::Expression(Expression { expr: Expr::Call(ref def_call), .. - }) = pipeline.expressions.get(0) + })) = pipeline.elements.get(0) { call = def_call.clone(); @@ -1063,10 +1062,10 @@ pub fn parse_export_in_module( }; // Trying to warp the 'alias' call into the 'export alias' in a very clumsy way - if let Some(Expression { + if let Some(PipelineElement::Expression(Expression { expr: Expr::Call(ref alias_call), .. - }) = pipeline.expressions.get(0) + })) = pipeline.elements.get(0) { call = alias_call.clone(); @@ -1129,10 +1128,10 @@ pub fn parse_export_in_module( }; // Trying to warp the 'use' call into the 'export use' in a very clumsy way - if let Some(Expression { + if let Some(PipelineElement::Expression(Expression { expr: Expr::Call(ref use_call), .. - }) = pipeline.expressions.get(0) + })) = pipeline.elements.get(0) { call = use_call.clone(); @@ -1314,11 +1313,9 @@ pub fn parse_module_block( for pipeline in &output.block { if pipeline.commands.len() == 1 { - parse_def_predecl( - working_set, - &pipeline.commands[0].parts, - expand_aliases_denylist, - ); + if let LiteElement::Command(command) = &pipeline.commands[0] { + parse_def_predecl(working_set, &command.parts, expand_aliases_denylist); + } } } @@ -1329,91 +1326,91 @@ pub fn parse_module_block( .iter() .map(|pipeline| { if pipeline.commands.len() == 1 { - let name = working_set.get_span_contents(pipeline.commands[0].parts[0]); + match &pipeline.commands[0] { + LiteElement::Command(command) => { + let name = working_set.get_span_contents(command.parts[0]); - let (pipeline, err) = match name { - b"def" | b"def-env" => { - let (pipeline, err) = - parse_def(working_set, &pipeline.commands[0], expand_aliases_denylist); + let (pipeline, err) = match name { + b"def" | b"def-env" => { + let (pipeline, err) = + parse_def(working_set, command, expand_aliases_denylist); - (pipeline, err) - } - b"extern" => { - let (pipeline, err) = parse_extern( - working_set, - &pipeline.commands[0], - expand_aliases_denylist, - ); + (pipeline, err) + } + b"extern" => { + let (pipeline, err) = + parse_extern(working_set, command, expand_aliases_denylist); - (pipeline, err) - } - b"alias" => { - let (pipeline, err) = parse_alias( - working_set, - &pipeline.commands[0].parts, - expand_aliases_denylist, - ); + (pipeline, err) + } + b"alias" => { + let (pipeline, err) = parse_alias( + working_set, + &command.parts, + expand_aliases_denylist, + ); - (pipeline, err) - } - b"use" => { - let (pipeline, _, err) = parse_use( - working_set, - &pipeline.commands[0].parts, - expand_aliases_denylist, - ); + (pipeline, err) + } + b"use" => { + let (pipeline, _, err) = + parse_use(working_set, &command.parts, expand_aliases_denylist); - (pipeline, err) - } - b"export" => { - let (pipe, exportables, err) = parse_export_in_module( - working_set, - &pipeline.commands[0], - expand_aliases_denylist, - ); + (pipeline, err) + } + b"export" => { + let (pipe, exportables, err) = parse_export_in_module( + working_set, + command, + expand_aliases_denylist, + ); - if err.is_none() { - for exportable in exportables { - match exportable { - Exportable::Decl { name, id } => { - module.add_decl(name, id); - } - Exportable::Alias { name, id } => { - module.add_alias(name, id); + if err.is_none() { + for exportable in exportables { + match exportable { + Exportable::Decl { name, id } => { + module.add_decl(name, id); + } + Exportable::Alias { name, id } => { + module.add_alias(name, id); + } + } } } + + (pipe, err) } + b"export-env" => { + let (pipe, maybe_env_block, err) = parse_export_env( + working_set, + &command.parts, + expand_aliases_denylist, + ); + + if let Some(block_id) = maybe_env_block { + module.add_env_block(block_id); + } + + (pipe, err) + } + _ => ( + garbage_pipeline(&command.parts), + Some(ParseError::ExpectedKeyword( + "def or export keyword".into(), + command.parts[0], + )), + ), + }; + if error.is_none() { + error = err; } - (pipe, err) + pipeline } - b"export-env" => { - let (pipe, maybe_env_block, err) = parse_export_env( - working_set, - &pipeline.commands[0].parts, - expand_aliases_denylist, - ); - - if let Some(block_id) = maybe_env_block { - module.add_env_block(block_id); - } - - (pipe, err) - } - _ => ( - garbage_pipeline(&pipeline.commands[0].parts), - Some(ParseError::ExpectedKeyword( - "def or export keyword".into(), - pipeline.commands[0].parts[0], - )), - ), - }; - - if error.is_none() { - error = err; + LiteElement::Or(command) + | LiteElement::And(command) + | LiteElement::Redirection(command) => (garbage_pipeline(&command.parts)), } - - pipeline } else { error = Some(ParseError::Expected("not a pipeline".into(), span)); garbage_pipeline(&[span]) @@ -2841,14 +2838,12 @@ pub fn parse_let( ); return ( - Pipeline { - expressions: vec![Expression { - expr: Expr::Call(call), - span: nu_protocol::span(spans), - ty: output, - custom_completion: None, - }], - }, + Pipeline::from_vec(vec![Expression { + expr: Expr::Call(call), + span: nu_protocol::span(spans), + ty: output, + custom_completion: None, + }]), err, ); } @@ -2964,14 +2959,12 @@ pub fn parse_mut( ); return ( - Pipeline { - expressions: vec![Expression { - expr: Expr::Call(call), - span: nu_protocol::span(spans), - ty: output, - custom_completion: None, - }], - }, + Pipeline::from_vec(vec![Expression { + expr: Expr::Call(call), + span: nu_protocol::span(spans), + ty: output, + custom_completion: None, + }]), err, ); } diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index f17e10b161..9b38a1aa20 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -1,16 +1,14 @@ use crate::{ - lex, lite_parse, - lite_parse::LiteCommand, - parse_mut, + lex, parse_mut, type_check::{math_result_type, type_compatible}, - LiteBlock, ParseError, Token, TokenContents, + ParseError, Token, TokenContents, }; use nu_protocol::{ ast::{ Argument, Assignment, Bits, Block, Boolean, Call, CellPath, Comparison, Expr, Expression, FullCellPath, ImportPattern, ImportPatternHead, ImportPatternMember, Math, Operator, - PathMember, Pipeline, RangeInclusion, RangeOperator, + PathMember, Pipeline, PipelineElement, RangeInclusion, RangeOperator, }, engine::StateWorkingSet, span, BlockId, Flag, PositionalArg, Signature, Span, Spanned, SyntaxShape, Type, Unit, VarId, @@ -1055,11 +1053,16 @@ pub fn parse_call( let (mut result, err) = parse_builtin_commands(working_set, &lite_command, &expand_aliases_denylist); - let mut result = result.expressions.remove(0); + let result = result.elements.remove(0); - result.replace_span(working_set, expansion_span, orig_span); + // If this is the first element in a pipeline, we know it has to be an expression + if let PipelineElement::Expression(mut result) = result { + result.replace_span(working_set, expansion_span, orig_span); - return (result, err); + return (result, err); + } else { + panic!("Internal error: first element of pipeline not an expression") + } } } @@ -1908,9 +1911,6 @@ pub fn parse_full_cell_path( let (output, err) = lex(source, span.start, &[b'\n', b'\r'], &[], true); error = error.or(err); - let (output, err) = lite_parse(&output); - error = error.or(err); - // Creating a Type scope to parse the new block. This will keep track of // the previous input type found in that block let (output, err) = @@ -1922,9 +1922,19 @@ pub fn parse_full_cell_path( let ty = output .pipelines .last() - .and_then(|Pipeline { expressions, .. }| expressions.last()) - .map(|expr| match expr.expr { - Expr::BinaryOp(..) => expr.ty.clone(), + .and_then(|Pipeline { elements, .. }| elements.last()) + .map(|element| match element { + PipelineElement::Expression(expr) + if matches!( + expr, + Expression { + expr: Expr::BinaryOp(..), + .. + } + ) => + { + expr.ty.clone() + } _ => working_set.type_scope.get_last_output(), }) .unwrap_or_else(|| working_set.type_scope.get_last_output()); @@ -3059,7 +3069,9 @@ pub fn parse_row_condition( // We have an expression, so let's convert this into a block. let mut block = Block::new(); let mut pipeline = Pipeline::new(); - pipeline.expressions.push(expression); + pipeline + .elements + .push(PipelineElement::Expression(expression)); block.pipelines.push(pipeline); @@ -3709,27 +3721,29 @@ pub fn parse_list_expression( for arg in &output.block[0].commands { let mut spans_idx = 0; - while spans_idx < arg.parts.len() { - let (arg, err) = parse_multispan_value( - working_set, - &arg.parts, - &mut spans_idx, - element_shape, - expand_aliases_denylist, - ); - error = error.or(err); + if let LiteElement::Command(command) = arg { + while spans_idx < command.parts.len() { + let (arg, err) = parse_multispan_value( + working_set, + &command.parts, + &mut spans_idx, + element_shape, + expand_aliases_denylist, + ); + error = error.or(err); - if let Some(ref ctype) = contained_type { - if *ctype != arg.ty { - contained_type = Some(Type::Any); + if let Some(ref ctype) = contained_type { + if *ctype != arg.ty { + contained_type = Some(Type::Any); + } + } else { + contained_type = Some(arg.ty.clone()); } - } else { - contained_type = Some(arg.ty.clone()); + + args.push(arg); + + spans_idx += 1; } - - args.push(arg); - - spans_idx += 1; } } } @@ -3799,68 +3813,84 @@ pub fn parse_table_expression( ) } _ => { - let mut table_headers = vec![]; + match &output.block[0].commands[0] { + LiteElement::Command(command) + | LiteElement::Redirection(command) + | LiteElement::And(command) + | LiteElement::Or(command) => { + let mut table_headers = vec![]; - let (headers, err) = parse_value( - working_set, - output.block[0].commands[0].parts[0], - &SyntaxShape::List(Box::new(SyntaxShape::Any)), - expand_aliases_denylist, - ); - error = error.or(err); + let (headers, err) = parse_value( + working_set, + command.parts[0], + &SyntaxShape::List(Box::new(SyntaxShape::Any)), + expand_aliases_denylist, + ); + error = error.or(err); - if let Expression { - expr: Expr::List(headers), - .. - } = headers - { - table_headers = headers; - } - - let mut rows = vec![]; - for part in &output.block[1].commands[0].parts { - let (values, err) = parse_value( - working_set, - *part, - &SyntaxShape::List(Box::new(SyntaxShape::Any)), - expand_aliases_denylist, - ); - error = error.or(err); - if let Expression { - expr: Expr::List(values), - span, - .. - } = values - { - match values.len().cmp(&table_headers.len()) { - std::cmp::Ordering::Less => { - error = error - .or(Some(ParseError::MissingColumns(table_headers.len(), span))) - } - std::cmp::Ordering::Equal => {} - std::cmp::Ordering::Greater => { - error = error.or_else(|| { - Some(ParseError::ExtraColumns( - table_headers.len(), - values[table_headers.len()].span, - )) - }) - } + if let Expression { + expr: Expr::List(headers), + .. + } = headers + { + table_headers = headers; } - rows.push(values); + match &output.block[1].commands[0] { + LiteElement::Command(command) + | LiteElement::Redirection(command) + | LiteElement::And(command) + | LiteElement::Or(command) => { + let mut rows = vec![]; + for part in &command.parts { + let (values, err) = parse_value( + working_set, + *part, + &SyntaxShape::List(Box::new(SyntaxShape::Any)), + expand_aliases_denylist, + ); + error = error.or(err); + if let Expression { + expr: Expr::List(values), + span, + .. + } = values + { + match values.len().cmp(&table_headers.len()) { + std::cmp::Ordering::Less => { + error = error.or(Some(ParseError::MissingColumns( + table_headers.len(), + span, + ))) + } + std::cmp::Ordering::Equal => {} + std::cmp::Ordering::Greater => { + error = error.or_else(|| { + Some(ParseError::ExtraColumns( + table_headers.len(), + values[table_headers.len()].span, + )) + }) + } + } + + rows.push(values); + } + } + + ( + Expression { + expr: Expr::Table(table_headers, rows), + span: original_span, + ty: Type::Table(vec![]), //FIXME + custom_completion: None, + }, + error, + ) + } + } } } - - ( - Expression { - expr: Expr::Table(table_headers, rows), - span: original_span, - ty: Type::Table(vec![]), //FIXME - custom_completion: None, - }, - error, - ) } } } @@ -3919,9 +3949,6 @@ pub fn parse_block_expression( _ => (None, 0), }; - let (output, err) = lite_parse(&output[amt_to_skip..]); - error = error.or(err); - // TODO: Finish this if let SyntaxShape::Closure(Some(v)) = shape { if let Some((sig, sig_span)) = &signature { @@ -3955,8 +3982,13 @@ pub fn parse_block_expression( } } - let (mut output, err) = - parse_block(working_set, &output, false, expand_aliases_denylist, false); + let (mut output, err) = parse_block( + working_set, + &output[amt_to_skip..], + false, + expand_aliases_denylist, + false, + ); error = error.or(err); if let Some(signature) = signature { @@ -4088,9 +4120,6 @@ pub fn parse_closure_expression( _ => (None, 0), }; - let (output, err) = lite_parse(&output[amt_to_skip..]); - error = error.or(err); - // TODO: Finish this if let SyntaxShape::Closure(Some(v)) = shape { if let Some((sig, sig_span)) = &signature { @@ -4124,8 +4153,13 @@ pub fn parse_closure_expression( } } - let (mut output, err) = - parse_block(working_set, &output, false, expand_aliases_denylist, false); + let (mut output, err) = parse_block( + working_set, + &output[amt_to_skip..], + false, + expand_aliases_denylist, + false, + ); error = error.or(err); if let Some(signature) = signature { @@ -4926,9 +4960,7 @@ pub fn parse_expression( if let Some(decl_id) = with_env { let mut block = Block::default(); let ty = output.ty.clone(); - block.pipelines = vec![Pipeline { - expressions: vec![output], - }]; + block.pipelines = vec![Pipeline::from_vec(vec![output])]; let block_id = working_set.add_block(block); @@ -5130,11 +5162,16 @@ pub fn parse_record( pub fn parse_block( working_set: &mut StateWorkingSet, - lite_block: &LiteBlock, + tokens: &[Token], scoped: bool, expand_aliases_denylist: &[usize], is_subexpression: bool, ) -> (Block, Option) { + let mut error = None; + + let (lite_block, err) = lite_parse(tokens); + error = error.or(err); + trace!("parsing block: {:?}", lite_block); if scoped { @@ -5142,18 +5179,21 @@ pub fn parse_block( } working_set.type_scope.enter_scope(); - let mut error = None; - // Pre-declare any definition so that definitions // that share the same block can see each other for pipeline in &lite_block.block { if pipeline.commands.len() == 1 { - if let Some(err) = parse_def_predecl( - working_set, - &pipeline.commands[0].parts, - expand_aliases_denylist, - ) { - error = error.or(Some(err)); + match &pipeline.commands[0] { + LiteElement::Command(command) + | LiteElement::Redirection(command) + | LiteElement::And(command) + | LiteElement::Or(command) => { + if let Some(err) = + parse_def_predecl(working_set, &command.parts, expand_aliases_denylist) + { + error = error.or(Some(err)); + } + } } } } @@ -5167,88 +5207,146 @@ pub fn parse_block( let mut output = pipeline .commands .iter() - .map(|command| { - let (expr, err) = - parse_expression(working_set, &command.parts, expand_aliases_denylist); + .map(|command| match command { + LiteElement::Command(command) => { + let (expr, err) = parse_expression( + working_set, + &command.parts, + expand_aliases_denylist, + ); - working_set.type_scope.add_type(expr.ty.clone()); + working_set.type_scope.add_type(expr.ty.clone()); + + if error.is_none() { + error = err; + } + + PipelineElement::Expression(expr) + } + LiteElement::Redirection(command) => { + let (expr, err) = parse_expression( + working_set, + &command.parts, + expand_aliases_denylist, + ); + + working_set.type_scope.add_type(expr.ty.clone()); + + if error.is_none() { + error = err; + } + + PipelineElement::Redirect(expr) + } + LiteElement::And(command) => { + let (expr, err) = parse_expression( + working_set, + &command.parts, + expand_aliases_denylist, + ); + + working_set.type_scope.add_type(expr.ty.clone()); + + if error.is_none() { + error = err; + } + + PipelineElement::And(expr) + } + LiteElement::Or(command) => { + let (expr, err) = parse_expression( + working_set, + &command.parts, + expand_aliases_denylist, + ); + + working_set.type_scope.add_type(expr.ty.clone()); + + if error.is_none() { + error = err; + } + + PipelineElement::Or(expr) + } + }) + .collect::>(); + + if is_subexpression { + for element in output.iter_mut().skip(1) { + if element.has_in_variable(working_set) { + *element = wrap_element_with_collect(working_set, element); + } + } + } else { + for element in output.iter_mut() { + if element.has_in_variable(working_set) { + *element = wrap_element_with_collect(working_set, element); + } + } + } + + Pipeline { elements: output } + } else { + match &pipeline.commands[0] { + LiteElement::Command(command) + | LiteElement::Redirection(command) + | LiteElement::And(command) + | LiteElement::Or(command) => { + let (mut pipeline, err) = + parse_builtin_commands(working_set, command, expand_aliases_denylist); + + if idx == 0 { + if let Some(let_decl_id) = working_set.find_decl(b"let", &Type::Any) { + if let Some(let_env_decl_id) = + working_set.find_decl(b"let-env", &Type::Any) + { + for element in pipeline.elements.iter_mut() { + if let PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) = element + { + if call.decl_id == let_decl_id + || call.decl_id == let_env_decl_id + { + // Do an expansion + if let Some(Expression { + expr: Expr::Keyword(_, _, expr), + .. + }) = call.positional_iter_mut().nth(1) + { + if expr.has_in_variable(working_set) { + *expr = Box::new(wrap_expr_with_collect( + working_set, + expr, + )); + } + } + continue; + } else if element.has_in_variable(working_set) + && !is_subexpression + { + *element = + wrap_element_with_collect(working_set, element); + } + } else if element.has_in_variable(working_set) + && !is_subexpression + { + *element = + wrap_element_with_collect(working_set, element); + } + } + } + } + } if error.is_none() { error = err; } - expr - }) - .collect::>(); - - if is_subexpression { - for expr in output.iter_mut().skip(1) { - if expr.has_in_variable(working_set) { - *expr = wrap_expr_with_collect(working_set, expr); - } - } - } else { - for expr in output.iter_mut() { - if expr.has_in_variable(working_set) { - *expr = wrap_expr_with_collect(working_set, expr); - } + pipeline } } - - Pipeline { - expressions: output, - } - } else { - let (mut pipeline, err) = parse_builtin_commands( - working_set, - &pipeline.commands[0], - expand_aliases_denylist, - ); - - if idx == 0 { - if let Some(let_decl_id) = working_set.find_decl(b"let", &Type::Any) { - if let Some(let_env_decl_id) = working_set.find_decl(b"let-env", &Type::Any) - { - for expr in pipeline.expressions.iter_mut() { - if let Expression { - expr: Expr::Call(call), - .. - } = expr - { - if call.decl_id == let_decl_id - || call.decl_id == let_env_decl_id - { - // Do an expansion - if let Some(Expression { - expr: Expr::Keyword(_, _, expr), - .. - }) = call.positional_iter_mut().nth(1) - { - if expr.has_in_variable(working_set) { - *expr = Box::new(wrap_expr_with_collect( - working_set, - expr, - )); - } - } - continue; - } else if expr.has_in_variable(working_set) && !is_subexpression - { - *expr = wrap_expr_with_collect(working_set, expr); - } - } else if expr.has_in_variable(working_set) && !is_subexpression { - *expr = wrap_expr_with_collect(working_set, expr); - } - } - } - } - } - - if error.is_none() { - error = err; - } - - pipeline } }) .into(); @@ -5306,14 +5404,32 @@ fn discover_captures_in_pipeline( seen_blocks: &mut HashMap>, ) -> Result, ParseError> { let mut output = vec![]; - for expr in &pipeline.expressions { - let result = discover_captures_in_expr(working_set, expr, seen, seen_blocks)?; + for element in &pipeline.elements { + let result = + discover_captures_in_pipeline_element(working_set, element, seen, seen_blocks)?; output.extend(&result); } Ok(output) } +// Closes over captured variables +pub fn discover_captures_in_pipeline_element( + working_set: &StateWorkingSet, + element: &PipelineElement, + seen: &mut Vec, + seen_blocks: &mut HashMap>, +) -> Result, ParseError> { + match element { + PipelineElement::Expression(expression) + | PipelineElement::Redirect(expression) + | PipelineElement::And(expression) + | PipelineElement::Or(expression) => { + discover_captures_in_expr(working_set, expression, seen, seen_blocks) + } + } +} + // Closes over captured variables pub fn discover_captures_in_expr( working_set: &StateWorkingSet, @@ -5553,6 +5669,26 @@ pub fn discover_captures_in_expr( Ok(output) } +fn wrap_element_with_collect( + working_set: &mut StateWorkingSet, + element: &PipelineElement, +) -> PipelineElement { + match element { + PipelineElement::Expression(expression) => { + PipelineElement::Expression(wrap_expr_with_collect(working_set, expression)) + } + PipelineElement::Redirect(expression) => { + PipelineElement::Redirect(wrap_expr_with_collect(working_set, expression)) + } + PipelineElement::And(expression) => { + PipelineElement::And(wrap_expr_with_collect(working_set, expression)) + } + PipelineElement::Or(expression) => { + PipelineElement::Or(wrap_expr_with_collect(working_set, expression)) + } + } +} + fn wrap_expr_with_collect(working_set: &mut StateWorkingSet, expr: &Expression) -> Expression { let span = expr.span; @@ -5573,9 +5709,7 @@ fn wrap_expr_with_collect(working_set: &mut StateWorkingSet, expr: &Expression) expr.replace_in_variable(working_set, var_id); let block = Block { - pipelines: vec![Pipeline { - expressions: vec![expr], - }], + pipelines: vec![Pipeline::from_vec(vec![expr])], signature: Box::new(signature), ..Default::default() }; @@ -5618,6 +5752,196 @@ fn wrap_expr_with_collect(working_set: &mut StateWorkingSet, expr: &Expression) } } +#[derive(Debug)] +pub struct LiteCommand { + pub comments: Vec, + pub parts: Vec, +} + +impl Default for LiteCommand { + fn default() -> Self { + Self::new() + } +} + +impl LiteCommand { + pub fn new() -> Self { + Self { + comments: vec![], + parts: vec![], + } + } + + pub fn push(&mut self, span: Span) { + self.parts.push(span); + } + + pub fn is_empty(&self) -> bool { + self.parts.is_empty() + } +} + +#[derive(Debug)] +pub enum LiteElement { + Command(LiteCommand), + Redirection(LiteCommand), + And(LiteCommand), + Or(LiteCommand), +} + +#[derive(Debug)] +pub struct LitePipeline { + pub commands: Vec, +} + +impl Default for LitePipeline { + fn default() -> Self { + Self::new() + } +} + +impl LitePipeline { + pub fn new() -> Self { + Self { commands: vec![] } + } + + pub fn push(&mut self, element: LiteElement) { + self.commands.push(element); + } + + pub fn is_empty(&self) -> bool { + self.commands.is_empty() + } +} + +#[derive(Debug)] +pub struct LiteBlock { + pub block: Vec, +} + +impl Default for LiteBlock { + fn default() -> Self { + Self::new() + } +} + +impl LiteBlock { + pub fn new() -> Self { + Self { block: vec![] } + } + + pub fn push(&mut self, pipeline: LitePipeline) { + self.block.push(pipeline); + } + + pub fn is_empty(&self) -> bool { + self.block.is_empty() + } +} + +pub fn lite_parse(tokens: &[Token]) -> (LiteBlock, Option) { + let mut block = LiteBlock::new(); + let mut curr_pipeline = LitePipeline::new(); + let mut curr_command = LiteCommand::new(); + + let mut last_token = TokenContents::Eol; + + let mut curr_comment: Option> = None; + + for token in tokens.iter() { + match &token.contents { + TokenContents::Item => { + // If we have a comment, go ahead and attach it + if let Some(curr_comment) = curr_comment.take() { + curr_command.comments = curr_comment; + } + curr_command.push(token.span); + last_token = TokenContents::Item; + } + TokenContents::Pipe => { + if !curr_command.is_empty() { + curr_pipeline.push(LiteElement::Command(curr_command)); + curr_command = LiteCommand::new(); + } + last_token = TokenContents::Pipe; + } + TokenContents::Eol => { + if last_token != TokenContents::Pipe { + if !curr_command.is_empty() { + curr_pipeline.push(LiteElement::Command(curr_command)); + + curr_command = LiteCommand::new(); + } + + if !curr_pipeline.is_empty() { + block.push(curr_pipeline); + + curr_pipeline = LitePipeline::new(); + } + } + + if last_token == TokenContents::Eol { + // Clear out the comment as we're entering a new comment + curr_comment = None; + } + + last_token = TokenContents::Eol; + } + TokenContents::Semicolon => { + if !curr_command.is_empty() { + curr_pipeline.push(LiteElement::Command(curr_command)); + + curr_command = LiteCommand::new(); + } + + if !curr_pipeline.is_empty() { + block.push(curr_pipeline); + + curr_pipeline = LitePipeline::new(); + } + + last_token = TokenContents::Semicolon; + } + TokenContents::Comment => { + // Comment is beside something + if last_token != TokenContents::Eol { + curr_command.comments.push(token.span); + curr_comment = None; + } else { + // Comment precedes something + if let Some(curr_comment) = &mut curr_comment { + curr_comment.push(token.span); + } else { + curr_comment = Some(vec![token.span]); + } + } + + last_token = TokenContents::Comment; + } + } + } + + if !curr_command.is_empty() { + curr_pipeline.push(LiteElement::Command(curr_command)); + } + + if !curr_pipeline.is_empty() { + block.push(curr_pipeline); + } + + if last_token == TokenContents::Pipe { + ( + block, + Some(ParseError::UnexpectedEof( + "pipeline missing end".into(), + tokens[tokens.len() - 1].span, + )), + ) + } else { + (block, None) + } +} + // Parses a vector of u8 to create an AST Block. If a file name is given, then // the name is stored in the working set. When parsing a source without a file // name, the source of bytes is stored as "source" @@ -5644,9 +5968,6 @@ pub fn parse( let (output, err) = lex(contents, span_offset, &[], &[], false); error = error.or(err); - let (output, err) = lite_parse(&output); - error = error.or(err); - let (mut output, err) = parse_block(working_set, &output, scoped, expand_aliases_denylist, false); error = error.or(err); diff --git a/crates/nu-parser/tests/test_lite_parser.rs b/crates/nu-parser/tests/test_lite_parser.rs deleted file mode 100644 index b92db98c46..0000000000 --- a/crates/nu-parser/tests/test_lite_parser.rs +++ /dev/null @@ -1,242 +0,0 @@ -use nu_parser::{lex, lite_parse, LiteBlock, ParseError}; -use nu_protocol::Span; - -fn lite_parse_helper(input: &[u8]) -> Result { - let (output, err) = lex(input, 0, &[], &[], false); - if let Some(err) = err { - return Err(err); - } - - let (output, err) = lite_parse(&output); - if let Some(err) = err { - return Err(err); - } - - Ok(output) -} - -#[test] -fn comment_before() -> Result<(), ParseError> { - // Code: - // # this is a comment - // def foo bar - let input = b"# this is a comment\ndef foo bar"; - - let lite_block = lite_parse_helper(input)?; - - assert_eq!(lite_block.block.len(), 1); - assert_eq!(lite_block.block[0].commands.len(), 1); - assert_eq!(lite_block.block[0].commands[0].comments.len(), 1); - assert_eq!(lite_block.block[0].commands[0].parts.len(), 3); - - assert_eq!( - lite_block.block[0].commands[0].comments[0], - Span { start: 0, end: 19 } - ); - - Ok(()) -} - -#[test] -fn comment_beside() -> Result<(), ParseError> { - // Code: - // def foo bar # this is a comment - let input = b"def foo bar # this is a comment"; - - let lite_block = lite_parse_helper(input)?; - - assert_eq!(lite_block.block.len(), 1); - assert_eq!(lite_block.block[0].commands.len(), 1); - assert_eq!(lite_block.block[0].commands[0].comments.len(), 1); - assert_eq!(lite_block.block[0].commands[0].parts.len(), 3); - - assert_eq!( - lite_block.block[0].commands[0].comments[0], - Span { start: 12, end: 31 } - ); - - Ok(()) -} - -#[test] -fn comments_stack() -> Result<(), ParseError> { - // Code: - // # this is a comment - // # another comment - // # def foo bar - let input = b"# this is a comment\n# another comment\ndef foo bar "; - - let lite_block = lite_parse_helper(input)?; - - assert_eq!(lite_block.block.len(), 1); - assert_eq!(lite_block.block[0].commands[0].comments.len(), 2); - assert_eq!(lite_block.block[0].commands[0].parts.len(), 3); - - assert_eq!( - lite_block.block[0].commands[0].comments[0], - Span { start: 0, end: 19 } - ); - - assert_eq!( - lite_block.block[0].commands[0].comments[1], - Span { start: 20, end: 37 } - ); - - Ok(()) -} - -#[test] -fn separated_comments_dont_stack() -> Result<(), ParseError> { - // Code: - // # this is a comment - // - // # another comment - // # def foo bar - let input = b"# this is a comment\n\n# another comment\ndef foo bar "; - - let lite_block = lite_parse_helper(input)?; - - assert_eq!(lite_block.block.len(), 1); - assert_eq!(lite_block.block[0].commands[0].comments.len(), 1); - assert_eq!(lite_block.block[0].commands[0].parts.len(), 3); - - assert_eq!( - lite_block.block[0].commands[0].comments[0], - Span { start: 21, end: 38 } - ); - - Ok(()) -} - -#[test] -fn multiple_pipelines() -> Result<(), ParseError> { - // Code: - // # A comment - // let a = ( 3 + ( - // 4 + - // 5 )) - // let b = 1 # comment - let input = b"# comment \n let a = ( 3 + (\n 4 + \n 5 )) \n let b = 1 # comment"; - - let lite_block = lite_parse_helper(input)?; - - assert_eq!(lite_block.block.len(), 2); - assert_eq!(lite_block.block[0].commands[0].comments.len(), 1); - assert_eq!(lite_block.block[0].commands[0].parts.len(), 4); - assert_eq!( - lite_block.block[0].commands[0].comments[0], - Span { start: 0, end: 10 } - ); - - assert_eq!(lite_block.block[1].commands[0].comments.len(), 1); - assert_eq!(lite_block.block[1].commands[0].parts.len(), 4); - assert_eq!( - lite_block.block[1].commands[0].comments[0], - Span { start: 52, end: 61 } - ); - - Ok(()) -} - -#[test] -fn multiple_commands() -> Result<(), ParseError> { - // Pipes add commands to the lite parser - // Code: - // let a = ls | where name == 1 - // let b = 1 # comment - let input = b"let a = ls | where name == 1 \n let b = 1 # comment"; - - let lite_block = lite_parse_helper(input)?; - - assert_eq!(lite_block.block.len(), 2); - assert_eq!(lite_block.block[0].commands.len(), 2); - assert_eq!(lite_block.block[1].commands.len(), 1); - - assert_eq!( - lite_block.block[1].commands[0].comments[0], - Span { start: 41, end: 50 } - ); - - Ok(()) -} - -#[test] -fn multiple_commands_with_comment() -> Result<(), ParseError> { - // Pipes add commands to the lite parser - // The comments are attached to the commands next to them - // Code: - // let a = ls | where name == 1 # comment - // let b = 1 # comment - //let a = ls | where name == 1 # comment \n let b = 1 # comment - let input = b"let a = ls | where name == 1 # comment\n let b = 1 # comment"; - - let lite_block = lite_parse_helper(input)?; - - assert_eq!(lite_block.block.len(), 2); - assert_eq!(lite_block.block[0].commands.len(), 2); - assert_eq!(lite_block.block[1].commands.len(), 1); - - assert_eq!( - lite_block.block[0].commands[1].comments[0], - Span { start: 29, end: 38 } - ); - - Ok(()) -} - -#[test] -fn multiple_commands_with_pipes() -> Result<(), ParseError> { - // The comments inside () get encapsulated in the whole item - // Code: - // # comment 1 - // # comment 2 - // let a = ( ls - // | where name =~ some # another comment - // | each { |file| rm file.name } # final comment - // ) - // # comment A - // let b = 0; - let input = b"# comment 1 -# comment 2 -let a = ( ls -| where name =~ some # another comment -| each { |file| rm file.name }) # final comment -# comment A -let b = 0 -"; - - let lite_block = lite_parse_helper(input)?; - - assert_eq!(lite_block.block.len(), 2); - assert_eq!(lite_block.block[0].commands[0].comments.len(), 3); - assert_eq!(lite_block.block[0].commands[0].parts.len(), 4); - - assert_eq!( - lite_block.block[0].commands[0].parts[3], - Span { - start: 32, - end: 107 - } - ); - - assert_eq!( - lite_block.block[0].commands[0].comments[2], - Span { - start: 108, - end: 123 - } - ); - - assert_eq!(lite_block.block[1].commands[0].comments.len(), 1); - assert_eq!(lite_block.block[1].commands[0].parts.len(), 4); - - assert_eq!( - lite_block.block[1].commands[0].comments[0], - Span { - start: 124, - end: 135 - } - ); - - Ok(()) -} diff --git a/crates/nu-parser/tests/test_parser.rs b/crates/nu-parser/tests/test_parser.rs index 31acae1b58..d67c246b08 100644 --- a/crates/nu-parser/tests/test_parser.rs +++ b/crates/nu-parser/tests/test_parser.rs @@ -1,7 +1,7 @@ use nu_parser::ParseError; use nu_parser::*; use nu_protocol::{ - ast::{Expr, Expression}, + ast::{Expr, Expression, PipelineElement}, engine::{Command, EngineState, Stack, StateWorkingSet}, Signature, SyntaxShape, }; @@ -54,10 +54,10 @@ pub fn parse_int() { assert!(expressions.len() == 1); assert!(matches!( expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::Int(3), .. - } + }) )) } @@ -72,7 +72,11 @@ pub fn parse_binary_with_hex_format() { assert!(block.len() == 1); let expressions = &block[0]; assert!(expressions.len() == 1); - assert_eq!(expressions[0].expr, Expr::Binary(vec![0x13])) + if let PipelineElement::Expression(expr) = &expressions[0] { + assert_eq!(expr.expr, Expr::Binary(vec![0x13])) + } else { + panic!("Not an expression") + } } #[test] @@ -86,7 +90,11 @@ pub fn parse_binary_with_incomplete_hex_format() { assert!(block.len() == 1); let expressions = &block[0]; assert!(expressions.len() == 1); - assert_eq!(expressions[0].expr, Expr::Binary(vec![0x03])) + if let PipelineElement::Expression(expr) = &expressions[0] { + assert_eq!(expr.expr, Expr::Binary(vec![0x03])) + } else { + panic!("Not an expression") + } } #[test] @@ -100,7 +108,11 @@ pub fn parse_binary_with_binary_format() { assert!(block.len() == 1); let expressions = &block[0]; assert!(expressions.len() == 1); - assert_eq!(expressions[0].expr, Expr::Binary(vec![0b10101000])) + if let PipelineElement::Expression(expr) = &expressions[0] { + assert_eq!(expr.expr, Expr::Binary(vec![0b10101000])) + } else { + panic!("Not an expression") + } } #[test] @@ -114,7 +126,11 @@ pub fn parse_binary_with_incomplete_binary_format() { assert!(block.len() == 1); let expressions = &block[0]; assert!(expressions.len() == 1); - assert_eq!(expressions[0].expr, Expr::Binary(vec![0b00000010])) + if let PipelineElement::Expression(expr) = &expressions[0] { + assert_eq!(expr.expr, Expr::Binary(vec![0b00000010])) + } else { + panic!("Not an expression") + } } #[test] @@ -128,7 +144,11 @@ pub fn parse_binary_with_octal_format() { assert!(block.len() == 1); let expressions = &block[0]; assert!(expressions.len() == 1); - assert_eq!(expressions[0].expr, Expr::Binary(vec![0o250])) + if let PipelineElement::Expression(expr) = &expressions[0] { + assert_eq!(expr.expr, Expr::Binary(vec![0o250])) + } else { + panic!("Not an expression") + } } #[test] @@ -142,7 +162,11 @@ pub fn parse_binary_with_incomplete_octal_format() { assert!(block.len() == 1); let expressions = &block[0]; assert!(expressions.len() == 1); - assert_eq!(expressions[0].expr, Expr::Binary(vec![0o2])) + if let PipelineElement::Expression(expr) = &expressions[0] { + assert_eq!(expr.expr, Expr::Binary(vec![0o2])) + } else { + panic!("Not an expression") + } } #[test] @@ -156,7 +180,11 @@ pub fn parse_binary_with_invalid_octal_format() { assert!(block.len() == 1); let expressions = &block[0]; assert!(expressions.len() == 1); - assert!(!matches!(&expressions[0].expr, Expr::Binary(_))) + if let PipelineElement::Expression(expr) = &expressions[0] { + assert!(!matches!(&expr.expr, Expr::Binary(_))) + } else { + panic!("Not an expression") + } } #[test] @@ -172,7 +200,11 @@ pub fn parse_binary_with_multi_byte_char() { assert!(block.len() == 1); let expressions = &block[0]; assert!(expressions.len() == 1); - assert!(!matches!(&expressions[0].expr, Expr::Binary(_))) + if let PipelineElement::Expression(expr) = &expressions[0] { + assert!(!matches!(&expr.expr, Expr::Binary(_))) + } else { + panic!("Not an expression") + } } #[test] @@ -186,10 +218,11 @@ pub fn parse_string() { assert!(block.len() == 1); let expressions = &block[0]; assert!(expressions.len() == 1); - assert_eq!( - expressions[0].expr, - Expr::String("hello nushell".to_string()) - ) + if let PipelineElement::Expression(expr) = &expressions[0] { + assert_eq!(expr.expr, Expr::String("hello nushell".to_string())) + } else { + panic!("Not an expression") + } } #[test] @@ -209,10 +242,11 @@ pub fn parse_escaped_string() { assert!(block.len() == 1); let expressions = &block[0]; assert!(expressions.len() == 1); - assert_eq!( - expressions[0].expr, - Expr::String("hello nushell".to_string()) - ) + if let PipelineElement::Expression(expr) = &expressions[0] { + assert_eq!(expr.expr, Expr::String("hello nushell".to_string())) + } else { + panic!("Not an expression") + } } #[test] @@ -231,10 +265,10 @@ pub fn parse_call() { let expressions = &block[0]; assert_eq!(expressions.len(), 1); - if let Expression { + if let PipelineElement::Expression(Expression { expr: Expr::Call(call), .. - } = &expressions[0] + }) = &expressions[0] { assert_eq!(call.decl_id, 0); } @@ -337,10 +371,10 @@ fn test_nothing_comparisson_eq() { assert!(expressions.len() == 1); assert!(matches!( &expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::BinaryOp(..), .. - } + }) )) } @@ -357,10 +391,10 @@ fn test_nothing_comparisson_neq() { assert!(expressions.len() == 1); assert!(matches!( &expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::BinaryOp(..), .. - } + }) )) } @@ -382,7 +416,7 @@ mod range { assert!(expressions.len() == 1); assert!(matches!( expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::Range( Some(_), None, @@ -393,7 +427,7 @@ mod range { } ), .. - } + }) )) } @@ -411,7 +445,7 @@ mod range { assert!(expressions.len() == 1); assert!(matches!( expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::Range( Some(_), None, @@ -422,7 +456,7 @@ mod range { } ), .. - } + }) )) } @@ -440,7 +474,7 @@ mod range { assert!(expressions.len() == 1); assert!(matches!( expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::Range( Some(_), None, @@ -451,7 +485,7 @@ mod range { } ), .. - } + }) )) } @@ -469,7 +503,7 @@ mod range { assert!(expressions.len() == 1); assert!(matches!( expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::Range( Some(_), None, @@ -480,7 +514,7 @@ mod range { } ), .. - } + }) )) } @@ -500,7 +534,7 @@ mod range { assert!(expressions.len() == 1); assert!(matches!( expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::Range( Some(_), None, @@ -511,7 +545,7 @@ mod range { } ), .. - } + }) )) } @@ -537,7 +571,7 @@ mod range { assert!(expressions.len() == 1); assert!(matches!( expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::Range( Some(_), None, @@ -548,7 +582,7 @@ mod range { } ), .. - } + }) )) } @@ -566,7 +600,7 @@ mod range { assert!(expressions.len() == 1); assert!(matches!( expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::Range( Some(_), None, @@ -577,7 +611,7 @@ mod range { } ), .. - } + }) )) } @@ -595,7 +629,7 @@ mod range { assert!(expressions.len() == 1); assert!(matches!( expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::Range( None, None, @@ -606,7 +640,7 @@ mod range { } ), .. - } + }) )) } @@ -624,7 +658,7 @@ mod range { assert!(expressions.len() == 1); assert!(matches!( expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::Range( Some(_), None, @@ -635,7 +669,7 @@ mod range { } ), .. - } + }) )) } @@ -653,7 +687,7 @@ mod range { assert!(expressions.len() == 1); assert!(matches!( expressions[0], - Expression { + PipelineElement::Expression(Expression { expr: Expr::Range( Some(_), Some(_), @@ -664,7 +698,7 @@ mod range { } ), .. - } + }) )) } @@ -997,8 +1031,11 @@ mod input_types { let expressions = &block[0]; assert!(expressions.len() == 3); - match &expressions[0].expr { - Expr::Call(call) => { + match &expressions[0] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let expected_id = working_set .find_decl(b"ls", &Type::Any) .expect("Error merging delta"); @@ -1007,16 +1044,22 @@ mod input_types { _ => panic!("Expected expression Call not found"), } - match &expressions[1].expr { - Expr::Call(call) => { + match &expressions[1] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let expected_id = working_set.find_decl(b"to-custom", &Type::Any).unwrap(); assert_eq!(call.decl_id, expected_id) } _ => panic!("Expected expression Call not found"), } - match &expressions[2].expr { - Expr::Call(call) => { + match &expressions[2] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let expected_id = working_set .find_decl(b"group-by", &Type::Custom("custom".into())) .unwrap(); @@ -1041,8 +1084,11 @@ mod input_types { assert!(block.len() == 3); let expressions = &block[2]; - match &expressions[1].expr { - Expr::Call(call) => { + match &expressions[1] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let expected_id = working_set .find_decl(b"agg", &Type::Custom("custom".into())) .unwrap(); @@ -1066,8 +1112,11 @@ mod input_types { assert!(block.len() == 2); let expressions = &block[1]; - match &expressions[1].expr { - Expr::Call(call) => { + match &expressions[1] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let expected_id = working_set .find_decl(b"agg", &Type::Custom("custom".into())) .unwrap(); @@ -1092,8 +1141,11 @@ mod input_types { assert!(block.len() == 3); let expressions = &block[1]; - match &expressions[1].expr { - Expr::Call(call) => { + match &expressions[1] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let expected_id = working_set.find_decl(b"to-custom", &Type::Any).unwrap(); assert_eq!(call.decl_id, expected_id) } @@ -1101,8 +1153,11 @@ mod input_types { } let expressions = &block[2]; - match &expressions[1].expr { - Expr::Call(call) => { + match &expressions[1] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let expected_id = working_set.find_decl(b"to-custom", &Type::Any).unwrap(); assert_eq!(call.decl_id, expected_id) } @@ -1126,16 +1181,22 @@ mod input_types { let expressions = &block[0]; assert!(expressions.len() == 2); - match &expressions[0].expr { - Expr::Call(call) => { + match &expressions[0] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let expected_id = working_set.find_decl(b"ls", &Type::Any).unwrap(); assert_eq!(call.decl_id, expected_id) } _ => panic!("Expected expression Call not found"), } - match &expressions[1].expr { - Expr::Call(call) => { + match &expressions[1] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let expected_id = working_set.find_decl(b"group-by", &Type::Any).unwrap(); assert_eq!(call.decl_id, expected_id) } @@ -1159,8 +1220,11 @@ mod input_types { engine_state.merge_delta(delta).unwrap(); let expressions = &block[0]; - match &expressions[3].expr { - Expr::Call(call) => { + match &expressions[3] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let arg = &call.arguments[0]; match arg { Argument::Positional(a) => match &a.expr { @@ -1171,8 +1235,11 @@ mod input_types { let expressions = &block[0]; assert!(expressions.len() == 2); - match &expressions[1].expr { - Expr::Call(call) => { + match &expressions[1] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let working_set = StateWorkingSet::new(&engine_state); let expected_id = working_set.find_decl(b"min", &Type::Any).unwrap(); @@ -1206,8 +1273,11 @@ mod input_types { assert!(block.len() == 1); let expressions = &block[0]; - match &expressions[2].expr { - Expr::Call(call) => { + match &expressions[2] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let expected_id = working_set .find_decl(b"with-column", &Type::Custom("custom".into())) .unwrap(); @@ -1216,8 +1286,11 @@ mod input_types { _ => panic!("Expected expression Call not found"), } - match &expressions[3].expr { - Expr::Call(call) => { + match &expressions[3] { + PipelineElement::Expression(Expression { + expr: Expr::Call(call), + .. + }) => { let expected_id = working_set .find_decl(b"collect", &Type::Custom("custom".into())) .unwrap(); diff --git a/crates/nu-protocol/src/ast/expression.rs b/crates/nu-protocol/src/ast/expression.rs index c4e6e77527..3f48ea74c8 100644 --- a/crates/nu-protocol/src/ast/expression.rs +++ b/crates/nu-protocol/src/ast/expression.rs @@ -134,8 +134,8 @@ impl Expression { } if let Some(pipeline) = block.pipelines.get(0) { - match pipeline.expressions.get(0) { - Some(expr) => expr.has_in_variable(working_set), + match pipeline.elements.get(0) { + Some(element) => element.has_in_variable(working_set), None => false, } } else { @@ -150,8 +150,8 @@ impl Expression { } if let Some(pipeline) = block.pipelines.get(0) { - match pipeline.expressions.get(0) { - Some(expr) => expr.has_in_variable(working_set), + match pipeline.elements.get(0) { + Some(element) => element.has_in_variable(working_set), None => false, } } else { @@ -256,7 +256,7 @@ impl Expression { let block = working_set.get_block(*block_id); if let Some(pipeline) = block.pipelines.get(0) { - if let Some(expr) = pipeline.expressions.get(0) { + if let Some(expr) = pipeline.elements.get(0) { expr.has_in_variable(working_set) } else { false @@ -302,10 +302,10 @@ impl Expression { let block = working_set.get_block(*block_id); let new_expr = if let Some(pipeline) = block.pipelines.get(0) { - if let Some(expr) = pipeline.expressions.get(0) { - let mut new_expr = expr.clone(); - new_expr.replace_in_variable(working_set, new_var_id); - Some(new_expr) + if let Some(element) = pipeline.elements.get(0) { + let mut new_element = element.clone(); + new_element.replace_in_variable(working_set, new_var_id); + Some(new_element) } else { None } @@ -317,7 +317,7 @@ impl Expression { if let Some(new_expr) = new_expr { if let Some(pipeline) = block.pipelines.get_mut(0) { - if let Some(expr) = pipeline.expressions.get_mut(0) { + if let Some(expr) = pipeline.elements.get_mut(0) { *expr = new_expr } } @@ -332,11 +332,11 @@ impl Expression { Expr::Closure(block_id) => { let block = working_set.get_block(*block_id); - let new_expr = if let Some(pipeline) = block.pipelines.get(0) { - if let Some(expr) = pipeline.expressions.get(0) { - let mut new_expr = expr.clone(); - new_expr.replace_in_variable(working_set, new_var_id); - Some(new_expr) + let new_element = if let Some(pipeline) = block.pipelines.get(0) { + if let Some(element) = pipeline.elements.get(0) { + let mut new_element = element.clone(); + new_element.replace_in_variable(working_set, new_var_id); + Some(new_element) } else { None } @@ -346,10 +346,10 @@ impl Expression { let block = working_set.get_block_mut(*block_id); - if let Some(new_expr) = new_expr { + if let Some(new_element) = new_element { if let Some(pipeline) = block.pipelines.get_mut(0) { - if let Some(expr) = pipeline.expressions.get_mut(0) { - *expr = new_expr + if let Some(element) = pipeline.elements.get_mut(0) { + *element = new_element } } } @@ -428,11 +428,11 @@ impl Expression { Expr::RowCondition(block_id) | Expr::Subexpression(block_id) => { let block = working_set.get_block(*block_id); - let new_expr = if let Some(pipeline) = block.pipelines.get(0) { - if let Some(expr) = pipeline.expressions.get(0) { - let mut new_expr = expr.clone(); - new_expr.replace_in_variable(working_set, new_var_id); - Some(new_expr) + let new_element = if let Some(pipeline) = block.pipelines.get(0) { + if let Some(element) = pipeline.elements.get(0) { + let mut new_element = element.clone(); + new_element.replace_in_variable(working_set, new_var_id); + Some(new_element) } else { None } @@ -442,10 +442,10 @@ impl Expression { let block = working_set.get_block_mut(*block_id); - if let Some(new_expr) = new_expr { + if let Some(new_element) = new_element { if let Some(pipeline) = block.pipelines.get_mut(0) { - if let Some(expr) = pipeline.expressions.get_mut(0) { - *expr = new_expr + if let Some(element) = pipeline.elements.get_mut(0) { + *element = new_element } } } @@ -499,8 +499,8 @@ impl Expression { let mut block = working_set.get_block(*block_id).clone(); for pipeline in block.pipelines.iter_mut() { - for expr in pipeline.expressions.iter_mut() { - expr.replace_span(working_set, replaced, new_span) + for element in pipeline.elements.iter_mut() { + element.replace_span(working_set, replaced, new_span) } } @@ -510,8 +510,8 @@ impl Expression { let mut block = working_set.get_block(*block_id).clone(); for pipeline in block.pipelines.iter_mut() { - for expr in pipeline.expressions.iter_mut() { - expr.replace_span(working_set, replaced, new_span) + for element in pipeline.elements.iter_mut() { + element.replace_span(working_set, replaced, new_span) } } @@ -589,8 +589,8 @@ impl Expression { let mut block = working_set.get_block(*block_id).clone(); for pipeline in block.pipelines.iter_mut() { - for expr in pipeline.expressions.iter_mut() { - expr.replace_span(working_set, replaced, new_span) + for element in pipeline.elements.iter_mut() { + element.replace_span(working_set, replaced, new_span) } } diff --git a/crates/nu-protocol/src/ast/pipeline.rs b/crates/nu-protocol/src/ast/pipeline.rs index 369e49a3a0..02614ca3af 100644 --- a/crates/nu-protocol/src/ast/pipeline.rs +++ b/crates/nu-protocol/src/ast/pipeline.rs @@ -1,10 +1,64 @@ use std::ops::{Index, IndexMut}; -use crate::ast::Expression; +use crate::{ast::Expression, engine::StateWorkingSet, Span, VarId}; + +#[derive(Debug, Clone)] +pub enum PipelineElement { + Expression(Expression), + Redirect(Expression), + And(Expression), + Or(Expression), +} + +impl PipelineElement { + pub fn span(&self) -> Span { + match self { + PipelineElement::Expression(expression) + | PipelineElement::Redirect(expression) + | PipelineElement::And(expression) + | PipelineElement::Or(expression) => expression.span, + } + } + pub fn has_in_variable(&self, working_set: &StateWorkingSet) -> bool { + match self { + PipelineElement::Expression(expression) + | PipelineElement::Redirect(expression) + | PipelineElement::And(expression) + | PipelineElement::Or(expression) => expression.has_in_variable(working_set), + } + } + + pub fn replace_in_variable(&mut self, working_set: &mut StateWorkingSet, new_var_id: VarId) { + match self { + PipelineElement::Expression(expression) + | PipelineElement::Redirect(expression) + | PipelineElement::And(expression) + | PipelineElement::Or(expression) => { + expression.replace_in_variable(working_set, new_var_id) + } + } + } + + pub fn replace_span( + &mut self, + working_set: &mut StateWorkingSet, + replaced: Span, + new_span: Span, + ) { + match self { + PipelineElement::Expression(expression) + | PipelineElement::Redirect(expression) + | PipelineElement::And(expression) + | PipelineElement::Or(expression) => { + expression.replace_span(working_set, replaced, new_span) + } + } + } +} #[derive(Debug, Clone)] pub struct Pipeline { - pub expressions: Vec, + pub elements: Vec, } impl Default for Pipeline { @@ -15,34 +69,37 @@ impl Default for Pipeline { impl Pipeline { pub fn new() -> Self { - Self { - expressions: vec![], - } + Self { elements: vec![] } } pub fn from_vec(expressions: Vec) -> Pipeline { - Self { expressions } + Self { + elements: expressions + .into_iter() + .map(PipelineElement::Expression) + .collect(), + } } pub fn len(&self) -> usize { - self.expressions.len() + self.elements.len() } pub fn is_empty(&self) -> bool { - self.expressions.is_empty() + self.elements.is_empty() } } impl Index for Pipeline { - type Output = Expression; + type Output = PipelineElement; fn index(&self, index: usize) -> &Self::Output { - &self.expressions[index] + &self.elements[index] } } impl IndexMut for Pipeline { fn index_mut(&mut self, index: usize) -> &mut Self::Output { - &mut self.expressions[index] + &mut self.elements[index] } } diff --git a/src/main.rs b/src/main.rs index 26df5ab233..e853352e60 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,7 +20,7 @@ use nu_engine::{get_full_help, CallExt}; use nu_parser::{escape_for_script_arg, escape_quote_string, parse}; use nu_path::canonicalize_with; use nu_protocol::{ - ast::{Call, Expr, Expression}, + ast::{Call, Expr, Expression, PipelineElement}, engine::{Command, EngineState, Stack, StateWorkingSet}, Category, Example, IntoPipelineData, PipelineData, RawStream, ShellError, Signature, Span, Spanned, SyntaxShape, Value, @@ -511,10 +511,10 @@ fn parse_commandline_args( // We should have a successful parse now if let Some(pipeline) = block.pipelines.get(0) { - if let Some(Expression { + if let Some(PipelineElement::Expression(Expression { expr: Expr::Call(call), .. - }) = pipeline.expressions.get(0) + })) = pipeline.elements.get(0) { let redirect_stdin = call.get_named_arg("stdin"); let login_shell = call.get_named_arg("login");