From c4c4d82bf44fee2a5c67134c4c054ffd39cf3517 Mon Sep 17 00:00:00 2001 From: JT Date: Thu, 2 Sep 2021 09:20:53 +1200 Subject: [PATCH 01/13] Try putting streams in Value --- TODO.md | 2 +- crates/nu-engine/src/eval.rs | 11 ++- crates/nu-engine/src/value.rs | 165 +++++++++++++++++++++++----------- src/main.rs | 4 +- 4 files changed, 122 insertions(+), 60 deletions(-) diff --git a/TODO.md b/TODO.md index 9637952187..2879da2b71 100644 --- a/TODO.md +++ b/TODO.md @@ -12,10 +12,10 @@ - [x] subcommand alias - [x] type inference from successful parse (eg not `List` but `List`) - [x] parsing tables +- [ ] Column path - [ ] ...rest without calling it rest - [ ] operator overflow - [ ] finish operator type-checking -- [ ] Column path - [ ] Ranges - [ ] Source - [ ] Autoenv diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index ea85f52f71..dab0bf846d 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -1,6 +1,9 @@ use std::time::Instant; -use crate::{state::State, value::Value}; +use crate::{ + state::State, + value::{IntoRowStream, IntoValueStream, Value}, +}; use nu_parser::{Block, Call, Expr, Expression, Operator, Span, Statement, Type}; #[derive(Debug)] @@ -124,7 +127,7 @@ fn eval_call(state: &State, call: &Call) -> Result { for expr in &call.positional { let val = eval_expression(state, expr)?; - output.push(val.to_string()); + output.push(val.into_string()); } Ok(Value::String { val: output.join(""), @@ -259,7 +262,7 @@ pub fn eval_expression(state: &State, expr: &Expression) -> Result Result>>); + +impl ValueStream { + pub fn into_string(self) -> String { + let val: Vec = self.collect(); + format!( + "[{}]", + val.into_iter() + .map(|x| x.into_string()) + .collect::>() + .join(", ".into()) + ) + } +} + +impl Debug for ValueStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ValueStream").finish() + } +} + +impl Iterator for ValueStream { + type Item = Value; + + fn next(&mut self) -> Option { + { + let mut iter = self.0.borrow_mut(); + iter.next() + } + } +} + +pub trait IntoValueStream { + fn into_value_stream(self) -> ValueStream; +} + +impl IntoValueStream for Vec { + fn into_value_stream(self) -> ValueStream { + ValueStream(Rc::new(RefCell::new(self.into_iter()))) + } +} + +#[derive(Clone)] +pub struct RowStream(Rc>>>); + +impl RowStream { + pub fn into_string(self, headers: Vec) -> String { + let val: Vec> = self.collect(); + format!( + "[{}]\n[{}]", + headers + .iter() + .map(|x| x.to_string()) + .collect::>() + .join(", ".into()), + val.into_iter() + .map(|x| { + x.into_iter() + .map(|x| x.into_string()) + .collect::>() + .join(", ".into()) + }) + .collect::>() + .join("\n") + ) + } +} + +impl Debug for RowStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ValueStream").finish() + } +} + +impl Iterator for RowStream { + type Item = Vec; + + fn next(&mut self) -> Option { + { + let mut iter = self.0.borrow_mut(); + iter.next() + } + } +} + +pub trait IntoRowStream { + fn into_row_stream(self) -> RowStream; +} + +impl IntoRowStream for Vec> { + fn into_row_stream(self) -> RowStream { + RowStream(Rc::new(RefCell::new(self.into_iter()))) + } +} + #[derive(Debug, Clone)] pub enum Value { Bool { @@ -23,12 +119,12 @@ pub enum Value { span: Span, }, List { - val: Vec, + val: ValueStream, span: Span, }, Table { headers: Vec, - val: Vec>, + val: RowStream, span: Span, }, Block { @@ -88,6 +184,19 @@ impl Value { Value::Block { .. } => Type::Block, } } + + pub fn into_string(self) -> String { + match self { + Value::Bool { val, .. } => val.to_string(), + Value::Int { val, .. } => val.to_string(), + Value::Float { val, .. } => val.to_string(), + Value::String { val, .. } => val, + Value::List { val, .. } => val.into_string(), + Value::Table { headers, val, .. } => val.into_string(headers), + Value::Block { val, .. } => format!("", val), + Value::Nothing { .. } => format!(""), + } + } } impl PartialEq for Value { @@ -97,62 +206,12 @@ impl PartialEq for Value { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => lhs == rhs, (Value::Float { val: lhs, .. }, Value::Float { val: rhs, .. }) => lhs == rhs, (Value::String { val: lhs, .. }, Value::String { val: rhs, .. }) => lhs == rhs, - (Value::List { val: l1, .. }, Value::List { val: l2, .. }) => l1 == l2, (Value::Block { val: b1, .. }, Value::Block { val: b2, .. }) => b1 == b2, _ => false, } } } -impl Display for Value { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Value::Bool { val, .. } => { - write!(f, "{}", val) - } - Value::Int { val, .. } => { - write!(f, "{}", val) - } - Value::Float { val, .. } => { - write!(f, "{}", val) - } - Value::String { val, .. } => write!(f, "{}", val), - Value::List { val, .. } => { - write!( - f, - "[{}]", - val.iter() - .map(|x| x.to_string()) - .collect::>() - .join(", ".into()) - ) - } - Value::Table { headers, val, .. } => { - write!( - f, - "[{}]\n[{}]", - headers - .iter() - .map(|x| x.to_string()) - .collect::>() - .join(", ".into()), - val.iter() - .map(|x| { - x.iter() - .map(|x| x.to_string()) - .collect::>() - .join(", ".into()) - }) - .collect::>() - .join("\n") - ) - } - Value::Block { .. } => write!(f, ""), - Value::Nothing { .. } => write!(f, ""), - } - } -} - impl Value { pub fn add(&self, op: Span, rhs: &Value) -> Result { let span = nu_parser::span(&[self.span(), rhs.span()]); diff --git a/src/main.rs b/src/main.rs index 789963ef3c..6e29c85948 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,7 +32,7 @@ fn main() -> std::io::Result<()> { match eval_block(&state, &block) { Ok(value) => { - println!("{}", value); + println!("{}", value.into_string()); } Err(err) => { let parser_state = parser_state.borrow(); @@ -94,7 +94,7 @@ fn main() -> std::io::Result<()> { match eval_block(&state, &block) { Ok(value) => { - println!("{}", value); + println!("{}", value.into_string()); } Err(err) => { let parser_state = parser_state.borrow(); From 3d252a9797d6e4abf7e6537f13cb74b14c5414e5 Mon Sep 17 00:00:00 2001 From: JT Date: Thu, 2 Sep 2021 13:29:43 +1200 Subject: [PATCH 02/13] Add nu-protocol --- Cargo.lock | 11 ++ Cargo.toml | 2 +- crates/nu-cli/Cargo.toml | 1 + crates/nu-cli/src/default_context.rs | 3 +- crates/nu-cli/src/errors.rs | 4 +- crates/nu-command/Cargo.toml | 8 + crates/nu-command/src/length.rs | 0 crates/nu-command/src/lib.rs | 7 + crates/nu-engine/Cargo.toml | 3 +- crates/nu-engine/src/eval.rs | 24 +-- crates/nu-engine/src/lib.rs | 4 +- crates/nu-engine/src/state.rs | 4 +- crates/nu-parser/Cargo.toml | 3 +- crates/nu-parser/src/errors.rs | 4 +- crates/nu-parser/src/flatten.rs | 3 +- crates/nu-parser/src/lex.rs | 5 +- crates/nu-parser/src/lib.rs | 13 +- crates/nu-parser/src/lite_parse.rs | 3 +- crates/nu-parser/src/parser.rs | 141 ++---------------- crates/nu-parser/src/parser_state.rs | 53 +------ crates/nu-parser/src/type_check.rs | 3 +- crates/nu-parser/tests/test_lex.rs | 3 +- crates/nu-parser/tests/test_lite_parser.rs | 3 +- crates/nu-parser/tests/test_parser.rs | 3 +- crates/nu-protocol/Cargo.toml | 8 + crates/nu-protocol/README.md | 3 + .../src/declaration.rs | 1 - crates/nu-protocol/src/id.rs | 3 + crates/nu-protocol/src/lib.rs | 17 +++ crates/nu-protocol/src/shell_error.rs | 17 +++ .../src/signature.rs | 3 +- crates/{nu-parser => nu-protocol}/src/span.rs | 15 ++ crates/nu-protocol/src/syntax_shape.rs | 104 +++++++++++++ crates/nu-protocol/src/ty.rs | 40 +++++ .../{nu-engine => nu-protocol}/src/value.rs | 24 +-- 35 files changed, 296 insertions(+), 247 deletions(-) create mode 100644 crates/nu-command/Cargo.toml create mode 100644 crates/nu-command/src/length.rs create mode 100644 crates/nu-command/src/lib.rs create mode 100644 crates/nu-protocol/Cargo.toml create mode 100644 crates/nu-protocol/README.md rename crates/{nu-parser => nu-protocol}/src/declaration.rs (84%) create mode 100644 crates/nu-protocol/src/id.rs create mode 100644 crates/nu-protocol/src/lib.rs create mode 100644 crates/nu-protocol/src/shell_error.rs rename crates/{nu-parser => nu-protocol}/src/signature.rs (99%) rename crates/{nu-parser => nu-protocol}/src/span.rs (60%) create mode 100644 crates/nu-protocol/src/syntax_shape.rs create mode 100644 crates/nu-protocol/src/ty.rs rename crates/{nu-engine => nu-protocol}/src/value.rs (96%) diff --git a/Cargo.lock b/Cargo.lock index bbec05e95a..2bde0dbf30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -292,14 +292,20 @@ dependencies = [ "nu-ansi-term", "nu-engine", "nu-parser", + "nu-protocol", "reedline", ] +[[package]] +name = "nu-command" +version = "0.1.0" + [[package]] name = "nu-engine" version = "0.1.0" dependencies = [ "nu-parser", + "nu-protocol", ] [[package]] @@ -320,8 +326,13 @@ name = "nu-parser" version = "0.1.0" dependencies = [ "codespan-reporting", + "nu-protocol", ] +[[package]] +name = "nu-protocol" +version = "0.1.0" + [[package]] name = "num-integer" version = "0.1.44" diff --git a/Cargo.toml b/Cargo.toml index 9bf10f63fb..73e2375eed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace] -members = ["crates/nu-cli", "crates/nu-engine", "crates/nu-parser"] +members = ["crates/nu-cli", "crates/nu-engine", "crates/nu-parser", "crates/nu-command", "crates/nu-protocol"] [dependencies] reedline = { git = "https://github.com/jntrnr/reedline", branch = "main" } diff --git a/crates/nu-cli/Cargo.toml b/crates/nu-cli/Cargo.toml index 115029612b..e5a8ba5e6f 100644 --- a/crates/nu-cli/Cargo.toml +++ b/crates/nu-cli/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" [dependencies] nu-engine = { path = "../nu-engine" } nu-parser = { path = "../nu-parser" } +nu-protocol = { path = "../nu-protocol" } codespan-reporting = "0.11.1" nu-ansi-term = "0.32.0" reedline = { git = "https://github.com/jntrnr/reedline", branch = "main" } diff --git a/crates/nu-cli/src/default_context.rs b/crates/nu-cli/src/default_context.rs index 459f04423c..d03de7ece8 100644 --- a/crates/nu-cli/src/default_context.rs +++ b/crates/nu-cli/src/default_context.rs @@ -1,6 +1,7 @@ use std::{cell::RefCell, rc::Rc}; -use nu_parser::{ParserState, ParserWorkingSet, Signature, SyntaxShape}; +use nu_parser::{ParserState, ParserWorkingSet}; +use nu_protocol::{Signature, SyntaxShape}; pub fn create_default_context() -> Rc> { let parser_state = Rc::new(RefCell::new(ParserState::new())); diff --git a/crates/nu-cli/src/errors.rs b/crates/nu-cli/src/errors.rs index e3923042df..b891c22c63 100644 --- a/crates/nu-cli/src/errors.rs +++ b/crates/nu-cli/src/errors.rs @@ -2,8 +2,8 @@ use core::ops::Range; use codespan_reporting::diagnostic::{Diagnostic, Label}; use codespan_reporting::term::termcolor::{ColorChoice, StandardStream}; -use nu_engine::ShellError; -use nu_parser::{ParseError, ParserWorkingSet, Span}; +use nu_parser::{ParseError, ParserWorkingSet}; +use nu_protocol::{ShellError, Span}; fn convert_span_to_diag( working_set: &ParserWorkingSet, diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml new file mode 100644 index 0000000000..eaaba9db15 --- /dev/null +++ b/crates/nu-command/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "nu-command" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/crates/nu-command/src/length.rs b/crates/nu-command/src/length.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/nu-command/src/lib.rs b/crates/nu-command/src/lib.rs new file mode 100644 index 0000000000..31e1bb209f --- /dev/null +++ b/crates/nu-command/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/crates/nu-engine/Cargo.toml b/crates/nu-engine/Cargo.toml index 95747a0dc2..3fbdb06501 100644 --- a/crates/nu-engine/Cargo.toml +++ b/crates/nu-engine/Cargo.toml @@ -4,4 +4,5 @@ version = "0.1.0" edition = "2018" [dependencies] -nu-parser = { path = "../nu-parser" } \ No newline at end of file +nu-parser = { path = "../nu-parser" } +nu-protocol = { path = "../nu-protocol" } \ No newline at end of file diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index dab0bf846d..6962e15416 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -1,26 +1,8 @@ use std::time::Instant; -use crate::{ - state::State, - value::{IntoRowStream, IntoValueStream, Value}, -}; -use nu_parser::{Block, Call, Expr, Expression, Operator, Span, Statement, Type}; - -#[derive(Debug)] -pub enum ShellError { - OperatorMismatch { - op_span: Span, - lhs_ty: Type, - lhs_span: Span, - rhs_ty: Type, - rhs_span: Span, - }, - Unsupported(Span), - InternalError(String), - VariableNotFound(Span), - CantConvert(String, Span), - DivisionByZero(Span), -} +use crate::state::State; +use nu_parser::{Block, Call, Expr, Expression, Operator, Statement}; +use nu_protocol::{IntoRowStream, IntoValueStream, ShellError, Span, Value}; pub fn eval_operator(op: &Expression) -> Result { match op { diff --git a/crates/nu-engine/src/lib.rs b/crates/nu-engine/src/lib.rs index 2d0db2a8e3..138fe7ba3c 100644 --- a/crates/nu-engine/src/lib.rs +++ b/crates/nu-engine/src/lib.rs @@ -1,7 +1,5 @@ mod eval; mod state; -mod value; -pub use eval::{eval_block, eval_expression, eval_operator, ShellError}; +pub use eval::{eval_block, eval_expression, eval_operator}; pub use state::{Stack, State}; -pub use value::Value; diff --git a/crates/nu-engine/src/state.rs b/crates/nu-engine/src/state.rs index e1a8305229..108111aa22 100644 --- a/crates/nu-engine/src/state.rs +++ b/crates/nu-engine/src/state.rs @@ -1,7 +1,7 @@ -use nu_parser::{ParserState, VarId}; +use nu_parser::ParserState; use std::{cell::RefCell, collections::HashMap, rc::Rc}; -use crate::{value::Value, ShellError}; +use nu_protocol::{ShellError, Value, VarId}; pub struct State { pub parser_state: Rc>, diff --git a/crates/nu-parser/Cargo.toml b/crates/nu-parser/Cargo.toml index 13f8ce4764..f0ca45a1fb 100644 --- a/crates/nu-parser/Cargo.toml +++ b/crates/nu-parser/Cargo.toml @@ -4,4 +4,5 @@ version = "0.1.0" edition = "2018" [dependencies] -codespan-reporting = "0.11.1" \ No newline at end of file +codespan-reporting = "0.11.1" +nu-protocol = { path = "../nu-protocol"} \ No newline at end of file diff --git a/crates/nu-parser/src/errors.rs b/crates/nu-parser/src/errors.rs index 3aee7d07d3..a084f98587 100644 --- a/crates/nu-parser/src/errors.rs +++ b/crates/nu-parser/src/errors.rs @@ -1,9 +1,7 @@ -use crate::parser_state::Type; use crate::ParserWorkingSet; +use nu_protocol::{Span, Type}; use std::ops::Range; -pub use crate::Span; - #[derive(Debug)] pub enum ParseError { ExtraTokens(Span), diff --git a/crates/nu-parser/src/flatten.rs b/crates/nu-parser/src/flatten.rs index d62beb0a26..a11887dd46 100644 --- a/crates/nu-parser/src/flatten.rs +++ b/crates/nu-parser/src/flatten.rs @@ -1,4 +1,5 @@ -use crate::{Block, Expr, Expression, ParserWorkingSet, Pipeline, Span, Statement}; +use crate::{Block, Expr, Expression, ParserWorkingSet, Pipeline, Statement}; +use nu_protocol::Span; #[derive(Debug)] pub enum FlatShape { diff --git a/crates/nu-parser/src/lex.rs b/crates/nu-parser/src/lex.rs index 5a33d87bb7..bc2d865b31 100644 --- a/crates/nu-parser/src/lex.rs +++ b/crates/nu-parser/src/lex.rs @@ -1,4 +1,5 @@ -use crate::{ParseError, Span}; +use crate::ParseError; +use nu_protocol::Span; #[derive(Debug, PartialEq, Eq)] pub enum TokenContents { @@ -307,4 +308,4 @@ pub fn lex( } } (output, error) -} \ No newline at end of file +} diff --git a/crates/nu-parser/src/lib.rs b/crates/nu-parser/src/lib.rs index 9c10dd18f8..9649078b51 100644 --- a/crates/nu-parser/src/lib.rs +++ b/crates/nu-parser/src/lib.rs @@ -1,23 +1,14 @@ -mod declaration; mod errors; mod flatten; mod lex; mod lite_parse; mod parser; mod parser_state; -mod signature; -mod span; mod type_check; -pub use declaration::Declaration; pub use errors::ParseError; pub use flatten::FlatShape; pub use lex::{lex, Token, TokenContents}; pub use lite_parse::{lite_parse, LiteBlock}; -pub use parser::{ - span, Block, Call, Expr, Expression, Import, Operator, Pipeline, Statement, SyntaxShape, - VarDecl, -}; -pub use parser_state::{BlockId, DeclId, ParserDelta, ParserState, ParserWorkingSet, Type, VarId}; -pub use signature::{Flag, PositionalArg, Signature}; -pub use span::Span; +pub use parser::{Block, Call, Expr, Expression, Import, Operator, Pipeline, Statement, VarDecl}; +pub use parser_state::{ParserDelta, ParserState, ParserWorkingSet}; diff --git a/crates/nu-parser/src/lite_parse.rs b/crates/nu-parser/src/lite_parse.rs index f702701b36..cd5d75931b 100644 --- a/crates/nu-parser/src/lite_parse.rs +++ b/crates/nu-parser/src/lite_parse.rs @@ -1,4 +1,5 @@ -use crate::{ParseError, Span, Token, TokenContents}; +use crate::{ParseError, Token, TokenContents}; +use nu_protocol::Span; #[derive(Debug)] pub struct LiteCommand { diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index 4e1a5281a3..e3e5c60936 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -3,117 +3,13 @@ use std::{ ops::{Index, IndexMut}, }; -use crate::{ - lex, lite_parse, - parser_state::{Type, VarId}, - signature::{Flag, PositionalArg}, - BlockId, DeclId, Declaration, LiteBlock, ParseError, ParserWorkingSet, Signature, Span, Token, - TokenContents, +use crate::{lex, lite_parse, LiteBlock, ParseError, ParserWorkingSet, Token, TokenContents}; + +use nu_protocol::{ + span, BlockId, DeclId, Declaration, Flag, PositionalArg, Signature, Span, SyntaxShape, Type, + VarId, }; -/// The syntactic shapes that values must match to be passed into a command. You can think of this as the type-checking that occurs when you call a function. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SyntaxShape { - /// A specific match to a word or symbol - Keyword(Vec, Box), - - /// Any syntactic form is allowed - Any, - - /// Strings and string-like bare words are allowed - String, - - /// A dotted path to navigate the table - ColumnPath, - - /// A dotted path to navigate the table (including variable) - FullColumnPath, - - /// Only a numeric (integer or decimal) value is allowed - Number, - - /// A range is allowed (eg, `1..3`) - Range, - - /// Only an integer value is allowed - Int, - - /// A filepath is allowed - FilePath, - - /// A glob pattern is allowed, eg `foo*` - GlobPattern, - - /// A block is allowed, eg `{start this thing}` - Block, - - /// A table is allowed, eg `[[first, second]; [1, 2]]` - Table, - - /// A table is allowed, eg `[first second]` - List(Box), - - /// A filesize value is allowed, eg `10kb` - Filesize, - - /// A duration value is allowed, eg `19day` - Duration, - - /// An operator - Operator, - - /// A math expression which expands shorthand forms on the lefthand side, eg `foo > 1` - /// The shorthand allows us to more easily reach columns inside of the row being passed in - RowCondition, - - /// A general math expression, eg `1 + 2` - MathExpression, - - /// A variable name - Variable, - - /// A variable with optional type, `x` or `x: int` - VarWithOptType, - - /// A signature for a definition, `[x:int, --foo]` - Signature, - - /// A general expression, eg `1 + 2` or `foo --bar` - Expression, -} - -impl SyntaxShape { - pub fn to_type(&self) -> Type { - match self { - SyntaxShape::Any => Type::Unknown, - SyntaxShape::Block => Type::Block, - SyntaxShape::ColumnPath => Type::Unknown, - SyntaxShape::Duration => Type::Duration, - SyntaxShape::Expression => Type::Unknown, - SyntaxShape::FilePath => Type::FilePath, - SyntaxShape::Filesize => Type::Filesize, - SyntaxShape::FullColumnPath => Type::Unknown, - SyntaxShape::GlobPattern => Type::String, - SyntaxShape::Int => Type::Int, - SyntaxShape::List(x) => { - let contents = x.to_type(); - Type::List(Box::new(contents)) - } - SyntaxShape::Keyword(_, expr) => expr.to_type(), - SyntaxShape::MathExpression => Type::Unknown, - SyntaxShape::Number => Type::Number, - SyntaxShape::Operator => Type::Unknown, - SyntaxShape::Range => Type::Unknown, - SyntaxShape::RowCondition => Type::Bool, - SyntaxShape::Signature => Type::Unknown, - SyntaxShape::String => Type::String, - SyntaxShape::Table => Type::Table, - SyntaxShape::VarWithOptType => Type::Unknown, - SyntaxShape::Variable => Type::Unknown, - } - } -} - #[derive(Debug, Clone, PartialEq, Eq)] pub enum Operator { Equal, @@ -404,21 +300,6 @@ fn check_call(command: Span, sig: &Signature, call: &Call) -> Option } } -pub fn span(spans: &[Span]) -> Span { - let length = spans.len(); - - if length == 0 { - Span::unknown() - } else if length == 1 { - spans[0] - } else { - Span { - start: spans[0].start, - end: spans[length - 1].end, - } - } -} - impl<'a> ParserWorkingSet<'a> { pub fn parse_external_call(&mut self, spans: &[Span]) -> (Expression, Option) { // TODO: add external parsing @@ -725,7 +606,7 @@ impl<'a> ParserWorkingSet<'a> { call.decl_id = decl_id; call.head = command_span; - let decl = self.get_decl(decl_id).clone(); + let signature = self.get_decl(decl_id).signature.clone(); // The index into the positional parameter in the definition let mut positional_idx = 0; @@ -738,8 +619,7 @@ impl<'a> ParserWorkingSet<'a> { let arg_span = spans[spans_idx]; // Check if we're on a long flag, if so, parse - let (long_name, arg, err) = - self.parse_long_flag(spans, &mut spans_idx, &decl.signature); + let (long_name, arg, err) = self.parse_long_flag(spans, &mut spans_idx, &signature); if let Some(long_name) = long_name { // We found a long flag, like --bar error = error.or(err); @@ -750,7 +630,7 @@ impl<'a> ParserWorkingSet<'a> { // Check if we're on a short flag or group of short flags, if so, parse let (short_flags, err) = - self.parse_short_flags(spans, &mut spans_idx, positional_idx, &decl.signature); + self.parse_short_flags(spans, &mut spans_idx, positional_idx, &signature); if let Some(short_flags) = short_flags { error = error.or(err); @@ -774,8 +654,9 @@ impl<'a> ParserWorkingSet<'a> { } // Parse a positional arg if there is one - if let Some(positional) = decl.signature.get_positional(positional_idx) { + if let Some(positional) = signature.get_positional(positional_idx) { //Make sure we leave enough spans for the remaining positionals + let decl = self.get_decl(decl_id); let end = self.calculate_end_span(&decl, spans, spans_idx, positional_idx); @@ -813,7 +694,7 @@ impl<'a> ParserWorkingSet<'a> { spans_idx += 1; } - let err = check_call(command_span, &decl.signature, &call); + let err = check_call(command_span, &signature, &call); error = error.or(err); // FIXME: type unknown diff --git a/crates/nu-parser/src/parser_state.rs b/crates/nu-parser/src/parser_state.rs index 6dafdb6336..b4d1c82d56 100644 --- a/crates/nu-parser/src/parser_state.rs +++ b/crates/nu-parser/src/parser_state.rs @@ -1,8 +1,8 @@ -use crate::{parser::Block, Declaration, Span}; +use crate::parser::Block; use core::panic; -use std::{collections::HashMap, fmt::Display, slice::Iter}; +use nu_protocol::{BlockId, DeclId, Declaration, Span, Type, VarId}; +use std::{collections::HashMap, slice::Iter}; -#[derive(Debug)] pub struct ParserState { files: Vec<(String, usize, usize)>, file_contents: Vec, @@ -12,49 +12,6 @@ pub struct ParserState { scope: Vec, } -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Type { - Int, - Float, - Bool, - String, - Block, - ColumnPath, - Duration, - FilePath, - Filesize, - List(Box), - Number, - Nothing, - Table, - Unknown, -} - -impl Display for Type { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Type::Block => write!(f, "block"), - Type::Bool => write!(f, "bool"), - Type::ColumnPath => write!(f, "column path"), - Type::Duration => write!(f, "duration"), - Type::FilePath => write!(f, "filepath"), - Type::Filesize => write!(f, "filesize"), - Type::Float => write!(f, "float"), - Type::Int => write!(f, "int"), - Type::List(l) => write!(f, "list<{}>", l), - Type::Nothing => write!(f, "nothing"), - Type::Number => write!(f, "number"), - Type::String => write!(f, "string"), - Type::Table => write!(f, "table"), - Type::Unknown => write!(f, "unknown"), - } - } -} - -pub type VarId = usize; -pub type DeclId = usize; -pub type BlockId = usize; - #[derive(Debug)] struct ScopeFrame { vars: HashMap, VarId>, @@ -136,7 +93,7 @@ impl ParserState { pub fn print_decls(&self) { for decl in self.decls.iter().enumerate() { - println!("decl{}: {:?}", decl.0, decl.1); + println!("decl{}: {:?}", decl.0, decl.1.signature); } } @@ -219,13 +176,11 @@ impl ParserState { } } -#[derive(Debug)] pub struct ParserWorkingSet<'a> { permanent_state: &'a ParserState, pub delta: ParserDelta, } -#[derive(Debug)] pub struct ParserDelta { files: Vec<(String, usize, usize)>, pub(crate) file_contents: Vec, diff --git a/crates/nu-parser/src/type_check.rs b/crates/nu-parser/src/type_check.rs index 9319e3d5f1..65b0697e96 100644 --- a/crates/nu-parser/src/type_check.rs +++ b/crates/nu-parser/src/type_check.rs @@ -1,4 +1,5 @@ -use crate::{parser::Operator, parser_state::Type, Expr, Expression, ParseError, ParserWorkingSet}; +use crate::{parser::Operator, Expr, Expression, ParseError, ParserWorkingSet}; +use nu_protocol::Type; impl<'a> ParserWorkingSet<'a> { pub fn type_compatible(lhs: &Type, rhs: &Type) -> bool { diff --git a/crates/nu-parser/tests/test_lex.rs b/crates/nu-parser/tests/test_lex.rs index 7646db7c86..a93088980b 100644 --- a/crates/nu-parser/tests/test_lex.rs +++ b/crates/nu-parser/tests/test_lex.rs @@ -1,4 +1,5 @@ -use nu_parser::{lex, ParseError, Span, Token, TokenContents}; +use nu_parser::{lex, ParseError, Token, TokenContents}; +use nu_protocol::Span; #[test] fn lex_basic() { diff --git a/crates/nu-parser/tests/test_lite_parser.rs b/crates/nu-parser/tests/test_lite_parser.rs index 81c415048f..9b739021ad 100644 --- a/crates/nu-parser/tests/test_lite_parser.rs +++ b/crates/nu-parser/tests/test_lite_parser.rs @@ -1,4 +1,5 @@ -use nu_parser::{lex, lite_parse, LiteBlock, ParseError, Span}; +use nu_parser::{lex, lite_parse, LiteBlock, ParseError}; +use nu_protocol::Span; fn lite_parse_helper(input: &[u8]) -> Result { let (output, err) = lex(input, 0, &[], &[]); diff --git a/crates/nu-parser/tests/test_parser.rs b/crates/nu-parser/tests/test_parser.rs index d524d422e1..ed38f3ccfd 100644 --- a/crates/nu-parser/tests/test_parser.rs +++ b/crates/nu-parser/tests/test_parser.rs @@ -1,5 +1,6 @@ use nu_parser::*; -use nu_parser::{ParseError, ParserState, Signature}; +use nu_parser::{ParseError, ParserState}; +use nu_protocol::{Signature, SyntaxShape}; #[test] pub fn parse_int() { diff --git a/crates/nu-protocol/Cargo.toml b/crates/nu-protocol/Cargo.toml new file mode 100644 index 0000000000..c1a398d04e --- /dev/null +++ b/crates/nu-protocol/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "nu-protocol" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/crates/nu-protocol/README.md b/crates/nu-protocol/README.md new file mode 100644 index 0000000000..9443d74fd5 --- /dev/null +++ b/crates/nu-protocol/README.md @@ -0,0 +1,3 @@ +# nu-protocol + +The nu-protocol crate holds the definitions of structs/traits that are used throughout Nushell. This gives us one way to expose them to many other crates, as well as make these definitions available to each other, without causing mutually recursive dependencies. \ No newline at end of file diff --git a/crates/nu-parser/src/declaration.rs b/crates/nu-protocol/src/declaration.rs similarity index 84% rename from crates/nu-parser/src/declaration.rs rename to crates/nu-protocol/src/declaration.rs index 585a1980ed..0bb638a33b 100644 --- a/crates/nu-parser/src/declaration.rs +++ b/crates/nu-protocol/src/declaration.rs @@ -1,6 +1,5 @@ use crate::{BlockId, Signature}; -#[derive(Clone, Debug)] pub struct Declaration { pub signature: Box, pub body: Option, diff --git a/crates/nu-protocol/src/id.rs b/crates/nu-protocol/src/id.rs new file mode 100644 index 0000000000..cf72289f43 --- /dev/null +++ b/crates/nu-protocol/src/id.rs @@ -0,0 +1,3 @@ +pub type VarId = usize; +pub type DeclId = usize; +pub type BlockId = usize; diff --git a/crates/nu-protocol/src/lib.rs b/crates/nu-protocol/src/lib.rs new file mode 100644 index 0000000000..919fc2adf0 --- /dev/null +++ b/crates/nu-protocol/src/lib.rs @@ -0,0 +1,17 @@ +mod declaration; +mod id; +mod shell_error; +mod signature; +mod span; +mod syntax_shape; +mod ty; +mod value; + +pub use declaration::*; +pub use id::*; +pub use shell_error::*; +pub use signature::*; +pub use span::*; +pub use syntax_shape::*; +pub use ty::*; +pub use value::*; diff --git a/crates/nu-protocol/src/shell_error.rs b/crates/nu-protocol/src/shell_error.rs new file mode 100644 index 0000000000..a0786d599b --- /dev/null +++ b/crates/nu-protocol/src/shell_error.rs @@ -0,0 +1,17 @@ +use crate::{Span, Type}; + +#[derive(Debug)] +pub enum ShellError { + OperatorMismatch { + op_span: Span, + lhs_ty: Type, + lhs_span: Span, + rhs_ty: Type, + rhs_span: Span, + }, + Unsupported(Span), + InternalError(String), + VariableNotFound(Span), + CantConvert(String, Span), + DivisionByZero(Span), +} diff --git a/crates/nu-parser/src/signature.rs b/crates/nu-protocol/src/signature.rs similarity index 99% rename from crates/nu-parser/src/signature.rs rename to crates/nu-protocol/src/signature.rs index f2e80d7511..f7e4ddea71 100644 --- a/crates/nu-parser/src/signature.rs +++ b/crates/nu-protocol/src/signature.rs @@ -1,4 +1,5 @@ -use crate::{parser::SyntaxShape, Declaration, VarId}; +use crate::VarId; +use crate::{Declaration, SyntaxShape}; #[derive(Debug, Clone)] pub struct Flag { diff --git a/crates/nu-parser/src/span.rs b/crates/nu-protocol/src/span.rs similarity index 60% rename from crates/nu-parser/src/span.rs rename to crates/nu-protocol/src/span.rs index 0777afef69..f1fd064355 100644 --- a/crates/nu-parser/src/span.rs +++ b/crates/nu-protocol/src/span.rs @@ -20,3 +20,18 @@ impl Span { } } } + +pub fn span(spans: &[Span]) -> Span { + let length = spans.len(); + + if length == 0 { + Span::unknown() + } else if length == 1 { + spans[0] + } else { + Span { + start: spans[0].start, + end: spans[length - 1].end, + } + } +} diff --git a/crates/nu-protocol/src/syntax_shape.rs b/crates/nu-protocol/src/syntax_shape.rs new file mode 100644 index 0000000000..c62048d17f --- /dev/null +++ b/crates/nu-protocol/src/syntax_shape.rs @@ -0,0 +1,104 @@ +use crate::Type; + +/// The syntactic shapes that values must match to be passed into a command. You can think of this as the type-checking that occurs when you call a function. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SyntaxShape { + /// A specific match to a word or symbol + Keyword(Vec, Box), + + /// Any syntactic form is allowed + Any, + + /// Strings and string-like bare words are allowed + String, + + /// A dotted path to navigate the table + ColumnPath, + + /// A dotted path to navigate the table (including variable) + FullColumnPath, + + /// Only a numeric (integer or decimal) value is allowed + Number, + + /// A range is allowed (eg, `1..3`) + Range, + + /// Only an integer value is allowed + Int, + + /// A filepath is allowed + FilePath, + + /// A glob pattern is allowed, eg `foo*` + GlobPattern, + + /// A block is allowed, eg `{start this thing}` + Block, + + /// A table is allowed, eg `[[first, second]; [1, 2]]` + Table, + + /// A table is allowed, eg `[first second]` + List(Box), + + /// A filesize value is allowed, eg `10kb` + Filesize, + + /// A duration value is allowed, eg `19day` + Duration, + + /// An operator + Operator, + + /// A math expression which expands shorthand forms on the lefthand side, eg `foo > 1` + /// The shorthand allows us to more easily reach columns inside of the row being passed in + RowCondition, + + /// A general math expression, eg `1 + 2` + MathExpression, + + /// A variable name + Variable, + + /// A variable with optional type, `x` or `x: int` + VarWithOptType, + + /// A signature for a definition, `[x:int, --foo]` + Signature, + + /// A general expression, eg `1 + 2` or `foo --bar` + Expression, +} + +impl SyntaxShape { + pub fn to_type(&self) -> Type { + match self { + SyntaxShape::Any => Type::Unknown, + SyntaxShape::Block => Type::Block, + SyntaxShape::ColumnPath => Type::Unknown, + SyntaxShape::Duration => Type::Duration, + SyntaxShape::Expression => Type::Unknown, + SyntaxShape::FilePath => Type::FilePath, + SyntaxShape::Filesize => Type::Filesize, + SyntaxShape::FullColumnPath => Type::Unknown, + SyntaxShape::GlobPattern => Type::String, + SyntaxShape::Int => Type::Int, + SyntaxShape::List(x) => { + let contents = x.to_type(); + Type::List(Box::new(contents)) + } + SyntaxShape::Keyword(_, expr) => expr.to_type(), + SyntaxShape::MathExpression => Type::Unknown, + SyntaxShape::Number => Type::Number, + SyntaxShape::Operator => Type::Unknown, + SyntaxShape::Range => Type::Unknown, + SyntaxShape::RowCondition => Type::Bool, + SyntaxShape::Signature => Type::Unknown, + SyntaxShape::String => Type::String, + SyntaxShape::Table => Type::Table, + SyntaxShape::VarWithOptType => Type::Unknown, + SyntaxShape::Variable => Type::Unknown, + } + } +} diff --git a/crates/nu-protocol/src/ty.rs b/crates/nu-protocol/src/ty.rs new file mode 100644 index 0000000000..80cbc3b5e6 --- /dev/null +++ b/crates/nu-protocol/src/ty.rs @@ -0,0 +1,40 @@ +use std::fmt::Display; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Type { + Int, + Float, + Bool, + String, + Block, + ColumnPath, + Duration, + FilePath, + Filesize, + List(Box), + Number, + Nothing, + Table, + Unknown, +} + +impl Display for Type { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Type::Block => write!(f, "block"), + Type::Bool => write!(f, "bool"), + Type::ColumnPath => write!(f, "column path"), + Type::Duration => write!(f, "duration"), + Type::FilePath => write!(f, "filepath"), + Type::Filesize => write!(f, "filesize"), + Type::Float => write!(f, "float"), + Type::Int => write!(f, "int"), + Type::List(l) => write!(f, "list<{}>", l), + Type::Nothing => write!(f, "nothing"), + Type::Number => write!(f, "number"), + Type::String => write!(f, "string"), + Type::Table => write!(f, "table"), + Type::Unknown => write!(f, "unknown"), + } + } +} diff --git a/crates/nu-engine/src/value.rs b/crates/nu-protocol/src/value.rs similarity index 96% rename from crates/nu-engine/src/value.rs rename to crates/nu-protocol/src/value.rs index 52cf01aaf0..bd328eb714 100644 --- a/crates/nu-engine/src/value.rs +++ b/crates/nu-protocol/src/value.rs @@ -1,6 +1,6 @@ use std::{cell::RefCell, fmt::Debug, rc::Rc}; -use nu_parser::{BlockId, Span, Type}; +use crate::{span, BlockId, Span, Type}; use crate::ShellError; @@ -194,7 +194,7 @@ impl Value { Value::List { val, .. } => val.into_string(), Value::Table { headers, val, .. } => val.into_string(headers), Value::Block { val, .. } => format!("", val), - Value::Nothing { .. } => format!(""), + Value::Nothing { .. } => String::new(), } } } @@ -214,7 +214,7 @@ impl PartialEq for Value { impl Value { pub fn add(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Int { @@ -248,7 +248,7 @@ impl Value { } } pub fn sub(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Int { @@ -278,7 +278,7 @@ impl Value { } } pub fn mul(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Int { @@ -308,7 +308,7 @@ impl Value { } } pub fn div(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => { @@ -362,7 +362,7 @@ impl Value { } } pub fn lt(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool { @@ -391,7 +391,7 @@ impl Value { } } pub fn lte(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool { @@ -420,7 +420,7 @@ impl Value { } } pub fn gt(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool { @@ -449,7 +449,7 @@ impl Value { } } pub fn gte(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool { @@ -478,7 +478,7 @@ impl Value { } } pub fn eq(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool { @@ -514,7 +514,7 @@ impl Value { } } pub fn ne(&self, op: Span, rhs: &Value) -> Result { - let span = nu_parser::span(&[self.span(), rhs.span()]); + let span = span(&[self.span(), rhs.span()]); match (self, rhs) { (Value::Int { val: lhs, .. }, Value::Int { val: rhs, .. }) => Ok(Value::Bool { From e1be8f61fcca459cc66269d988dd3be788de3bd4 Mon Sep 17 00:00:00 2001 From: JT Date: Thu, 2 Sep 2021 20:25:22 +1200 Subject: [PATCH 03/13] WIP --- Cargo.lock | 4 + Cargo.toml | 1 + crates/nu-cli/src/default_context.rs | 15 +- crates/nu-cli/src/errors.rs | 10 +- crates/nu-cli/src/syntax_highlight.rs | 11 +- crates/nu-engine/src/eval.rs | 34 +- crates/nu-engine/src/lib.rs | 4 + crates/nu-engine/src/state.rs | 6 +- crates/nu-parser/src/errors.rs | 95 +- crates/nu-parser/src/flatten.rs | 199 +- crates/nu-parser/src/lib.rs | 4 +- crates/nu-parser/src/parser.rs | 4182 ++++++++--------- crates/nu-parser/src/type_check.rs | 486 +- crates/nu-parser/tests/test_parser.rs | 40 +- crates/nu-protocol/Cargo.toml | 1 + crates/nu-protocol/src/block.rs | 44 + crates/nu-protocol/src/call.rs | 27 + crates/nu-protocol/src/command.rs | 60 + crates/nu-protocol/src/declaration.rs | 6 - .../src/engine_state.rs} | 179 +- crates/nu-protocol/src/example.rs | 7 + crates/nu-protocol/src/expr.rs | 21 + crates/nu-protocol/src/expression.rs | 85 + crates/nu-protocol/src/lib.rs | 22 +- crates/nu-protocol/src/operator.rs | 48 + crates/nu-protocol/src/pipeline.rs | 20 + crates/nu-protocol/src/signature.rs | 70 +- crates/nu-protocol/src/statement.rs | 8 + src/main.rs | 30 +- 29 files changed, 2965 insertions(+), 2754 deletions(-) create mode 100644 crates/nu-protocol/src/block.rs create mode 100644 crates/nu-protocol/src/call.rs create mode 100644 crates/nu-protocol/src/command.rs delete mode 100644 crates/nu-protocol/src/declaration.rs rename crates/{nu-parser/src/parser_state.rs => nu-protocol/src/engine_state.rs} (72%) create mode 100644 crates/nu-protocol/src/example.rs create mode 100644 crates/nu-protocol/src/expr.rs create mode 100644 crates/nu-protocol/src/expression.rs create mode 100644 crates/nu-protocol/src/operator.rs create mode 100644 crates/nu-protocol/src/pipeline.rs create mode 100644 crates/nu-protocol/src/statement.rs diff --git a/Cargo.lock b/Cargo.lock index 2bde0dbf30..2112588bf0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,6 +164,7 @@ dependencies = [ "nu-cli", "nu-engine", "nu-parser", + "nu-protocol", "pretty_assertions", "reedline", "tempfile", @@ -332,6 +333,9 @@ dependencies = [ [[package]] name = "nu-protocol" version = "0.1.0" +dependencies = [ + "codespan-reporting", +] [[package]] name = "num-integer" diff --git a/Cargo.toml b/Cargo.toml index 73e2375eed..6f1ff009f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ codespan-reporting = "0.11.1" nu-cli = { path="./crates/nu-cli" } nu-engine = { path="./crates/nu-engine" } nu-parser = { path="./crates/nu-parser" } +nu-protocol = { path = "./crates/nu-protocol" } # mimalloc = { version = "*", default-features = false } diff --git a/crates/nu-cli/src/default_context.rs b/crates/nu-cli/src/default_context.rs index d03de7ece8..4e349ca52c 100644 --- a/crates/nu-cli/src/default_context.rs +++ b/crates/nu-cli/src/default_context.rs @@ -1,13 +1,12 @@ use std::{cell::RefCell, rc::Rc}; -use nu_parser::{ParserState, ParserWorkingSet}; -use nu_protocol::{Signature, SyntaxShape}; +use nu_protocol::{EngineState, Signature, StateWorkingSet, SyntaxShape}; -pub fn create_default_context() -> Rc> { - let parser_state = Rc::new(RefCell::new(ParserState::new())); +pub fn create_default_context() -> Rc> { + let engine_state = Rc::new(RefCell::new(EngineState::new())); let delta = { - let parser_state = parser_state.borrow(); - let mut working_set = ParserWorkingSet::new(&*parser_state); + let engine_state = engine_state.borrow(); + let mut working_set = StateWorkingSet::new(&*engine_state); let sig = Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition"); @@ -109,8 +108,8 @@ pub fn create_default_context() -> Rc> { }; { - ParserState::merge_delta(&mut *parser_state.borrow_mut(), delta); + EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta); } - parser_state + engine_state } diff --git a/crates/nu-cli/src/errors.rs b/crates/nu-cli/src/errors.rs index b891c22c63..7115dc174a 100644 --- a/crates/nu-cli/src/errors.rs +++ b/crates/nu-cli/src/errors.rs @@ -2,11 +2,11 @@ use core::ops::Range; use codespan_reporting::diagnostic::{Diagnostic, Label}; use codespan_reporting::term::termcolor::{ColorChoice, StandardStream}; -use nu_parser::{ParseError, ParserWorkingSet}; -use nu_protocol::{ShellError, Span}; +use nu_parser::ParseError; +use nu_protocol::{ShellError, Span, StateWorkingSet}; fn convert_span_to_diag( - working_set: &ParserWorkingSet, + working_set: &StateWorkingSet, span: &Span, ) -> Result<(usize, Range), Box> { for (file_id, (_, start, end)) in working_set.files().enumerate() { @@ -22,7 +22,7 @@ fn convert_span_to_diag( } pub fn report_parsing_error( - working_set: &ParserWorkingSet, + working_set: &StateWorkingSet, error: &ParseError, ) -> Result<(), Box> { let writer = StandardStream::stderr(ColorChoice::Always); @@ -236,7 +236,7 @@ pub fn report_parsing_error( } pub fn report_shell_error( - working_set: &ParserWorkingSet, + working_set: &StateWorkingSet, error: &ShellError, ) -> Result<(), Box> { let writer = StandardStream::stderr(ColorChoice::Always); diff --git a/crates/nu-cli/src/syntax_highlight.rs b/crates/nu-cli/src/syntax_highlight.rs index cd0f592b8b..ac58ff9068 100644 --- a/crates/nu-cli/src/syntax_highlight.rs +++ b/crates/nu-cli/src/syntax_highlight.rs @@ -1,21 +1,22 @@ use nu_ansi_term::Style; -use nu_parser::{FlatShape, ParserState, ParserWorkingSet}; +use nu_parser::FlatShape; +use nu_protocol::{EngineState, StateWorkingSet}; use reedline::{Highlighter, StyledText}; use std::{cell::RefCell, rc::Rc}; pub struct NuHighlighter { - pub parser_state: Rc>, + pub engine_state: Rc>, } impl Highlighter for NuHighlighter { fn highlight(&self, line: &str) -> StyledText { let (shapes, global_span_offset) = { - let parser_state = self.parser_state.borrow(); - let mut working_set = ParserWorkingSet::new(&*parser_state); + let engine_state = self.engine_state.borrow(); + let mut working_set = StateWorkingSet::new(&*engine_state); let (block, _) = working_set.parse_source(line.as_bytes(), false); let shapes = working_set.flatten_block(&block); - (shapes, parser_state.next_span_start()) + (shapes, engine_state.next_span_start()) }; let mut output = StyledText::default(); diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index 6962e15416..5013c39955 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -1,7 +1,7 @@ use std::time::Instant; use crate::state::State; -use nu_parser::{Block, Call, Expr, Expression, Operator, Statement}; +use nu_protocol::{Block, Call, Expr, Expression, Operator, Statement}; use nu_protocol::{IntoRowStream, IntoValueStream, ShellError, Span, Value}; pub fn eval_operator(op: &Expression) -> Result { @@ -15,8 +15,8 @@ pub fn eval_operator(op: &Expression) -> Result { } fn eval_call(state: &State, call: &Call) -> Result { - let parser_state = state.parser_state.borrow(); - let decl = parser_state.get_decl(call.decl_id); + let engine_state = state.engine_state.borrow(); + let decl = engine_state.get_decl(call.decl_id); if let Some(block_id) = decl.body { let state = state.enter_scope(); for (arg, param) in call.positional.iter().zip( @@ -32,8 +32,8 @@ fn eval_call(state: &State, call: &Call) -> Result { state.add_var(var_id, result); } - let parser_state = state.parser_state.borrow(); - let block = parser_state.get_block(block_id); + let engine_state = state.engine_state.borrow(); + let block = engine_state.get_block(block_id); eval_block(&state, block) } else if decl.signature.name == "let" { let var_id = call.positional[0] @@ -80,15 +80,15 @@ fn eval_call(state: &State, call: &Call) -> Result { let result = eval_expression(state, cond)?; match result { Value::Bool { val, span } => { - let parser_state = state.parser_state.borrow(); + let engine_state = state.engine_state.borrow(); if val { - let block = parser_state.get_block(then_block); + let block = engine_state.get_block(then_block); let state = state.enter_scope(); eval_block(&state, block) } else if let Some(else_case) = else_case { if let Some(else_expr) = else_case.as_keyword() { if let Some(block_id) = else_expr.as_block() { - let block = parser_state.get_block(block_id); + let block = engine_state.get_block(block_id); let state = state.enter_scope(); eval_block(&state, block) } else { @@ -119,8 +119,8 @@ fn eval_call(state: &State, call: &Call) -> Result { let block = call.positional[0] .as_block() .expect("internal error: expected block"); - let parser_state = state.parser_state.borrow(); - let block = parser_state.get_block(block); + let engine_state = state.engine_state.borrow(); + let block = engine_state.get_block(block); let state = state.enter_scope(); let start_time = Instant::now(); @@ -143,8 +143,8 @@ fn eval_call(state: &State, call: &Call) -> Result { let block = call.positional[2] .as_block() .expect("internal error: expected block"); - let parser_state = state.parser_state.borrow(); - let block = parser_state.get_block(block); + let engine_state = state.engine_state.borrow(); + let block = engine_state.get_block(block); let state = state.enter_scope(); @@ -168,13 +168,13 @@ fn eval_call(state: &State, call: &Call) -> Result { span: call.positional[0].span, }) } else if decl.signature.name == "vars" { - state.parser_state.borrow().print_vars(); + state.engine_state.borrow().print_vars(); Ok(Value::Nothing { span: call.head }) } else if decl.signature.name == "decls" { - state.parser_state.borrow().print_decls(); + state.engine_state.borrow().print_decls(); Ok(Value::Nothing { span: call.head }) } else if decl.signature.name == "blocks" { - state.parser_state.borrow().print_blocks(); + state.engine_state.borrow().print_blocks(); Ok(Value::Nothing { span: call.head }) } else if decl.signature.name == "stack" { state.print_stack(); @@ -228,8 +228,8 @@ pub fn eval_expression(state: &State, expr: &Expression) -> Result { - let parser_state = state.parser_state.borrow(); - let block = parser_state.get_block(*block_id); + let engine_state = state.engine_state.borrow(); + let block = engine_state.get_block(*block_id); let state = state.enter_scope(); eval_block(&state, block) diff --git a/crates/nu-engine/src/lib.rs b/crates/nu-engine/src/lib.rs index 138fe7ba3c..4143f4ae64 100644 --- a/crates/nu-engine/src/lib.rs +++ b/crates/nu-engine/src/lib.rs @@ -1,5 +1,9 @@ +mod command; mod eval; +mod example; mod state; +pub use command::Command; pub use eval::{eval_block, eval_expression, eval_operator}; +pub use example::Example; pub use state::{Stack, State}; diff --git a/crates/nu-engine/src/state.rs b/crates/nu-engine/src/state.rs index 108111aa22..e9b5cff4ff 100644 --- a/crates/nu-engine/src/state.rs +++ b/crates/nu-engine/src/state.rs @@ -1,10 +1,10 @@ -use nu_parser::ParserState; +use nu_protocol::EngineState; use std::{cell::RefCell, collections::HashMap, rc::Rc}; use nu_protocol::{ShellError, Value, VarId}; pub struct State { - pub parser_state: Rc>, + pub engine_state: Rc>, pub stack: Stack, } @@ -15,7 +15,7 @@ impl State { pub fn enter_scope(&self) -> State { Self { - parser_state: self.parser_state.clone(), + engine_state: self.engine_state.clone(), stack: self.stack.clone().enter_scope(), } } diff --git a/crates/nu-parser/src/errors.rs b/crates/nu-parser/src/errors.rs index a084f98587..db82f95008 100644 --- a/crates/nu-parser/src/errors.rs +++ b/crates/nu-parser/src/errors.rs @@ -1,5 +1,4 @@ -use crate::ParserWorkingSet; -use nu_protocol::{Span, Type}; +use nu_protocol::{Span, StateWorkingSet, Type}; use std::ops::Range; #[derive(Debug)] @@ -31,95 +30,3 @@ pub enum ParseError { IncompleteParser(Span), RestNeedsName(Span), } - -impl<'a> codespan_reporting::files::Files<'a> for ParserWorkingSet<'a> { - type FileId = usize; - - type Name = String; - - type Source = String; - - fn name(&'a self, id: Self::FileId) -> Result { - Ok(self.get_filename(id)) - } - - fn source( - &'a self, - id: Self::FileId, - ) -> Result { - Ok(self.get_file_source(id)) - } - - fn line_index( - &'a self, - id: Self::FileId, - byte_index: usize, - ) -> Result { - let source = self.get_file_source(id); - - let mut count = 0; - - for byte in source.bytes().enumerate() { - if byte.0 == byte_index { - // println!("count: {} for file: {} index: {}", count, id, byte_index); - return Ok(count); - } - if byte.1 == b'\n' { - count += 1; - } - } - - // println!("count: {} for file: {} index: {}", count, id, byte_index); - Ok(count) - } - - fn line_range( - &'a self, - id: Self::FileId, - line_index: usize, - ) -> Result, codespan_reporting::files::Error> { - let source = self.get_file_source(id); - - let mut count = 0; - - let mut start = Some(0); - let mut end = None; - - for byte in source.bytes().enumerate() { - #[allow(clippy::comparison_chain)] - if count > line_index { - let start = start.expect("internal error: couldn't find line"); - let end = end.expect("internal error: couldn't find line"); - - // println!( - // "Span: {}..{} for fileid: {} index: {}", - // start, end, id, line_index - // ); - return Ok(start..end); - } else if count == line_index { - end = Some(byte.0 + 1); - } - - #[allow(clippy::comparison_chain)] - if byte.1 == b'\n' { - count += 1; - if count > line_index { - break; - } else if count == line_index { - start = Some(byte.0 + 1); - } - } - } - - match (start, end) { - (Some(start), Some(end)) => { - // println!( - // "Span: {}..{} for fileid: {} index: {}", - // start, end, id, line_index - // ); - Ok(start..end) - } - _ => Err(codespan_reporting::files::Error::FileMissing), - } - } -} diff --git a/crates/nu-parser/src/flatten.rs b/crates/nu-parser/src/flatten.rs index a11887dd46..dea7795c5b 100644 --- a/crates/nu-parser/src/flatten.rs +++ b/crates/nu-parser/src/flatten.rs @@ -1,5 +1,5 @@ -use crate::{Block, Expr, Expression, ParserWorkingSet, Pipeline, Statement}; use nu_protocol::Span; +use nu_protocol::{Block, Expr, Expression, Pipeline, StateWorkingSet, Statement}; #[derive(Debug)] pub enum FlatShape { @@ -16,101 +16,110 @@ pub enum FlatShape { Variable, } -impl<'a> ParserWorkingSet<'a> { - pub fn flatten_block(&self, block: &Block) -> Vec<(Span, FlatShape)> { - let mut output = vec![]; - for stmt in &block.stmts { - output.extend(self.flatten_statement(stmt)); - } - output +pub fn flatten_block(working_set: &StateWorkingSet, block: &Block) -> Vec<(Span, FlatShape)> { + let mut output = vec![]; + for stmt in &block.stmts { + output.extend(flatten_statement(working_set, stmt)); } + output +} - pub fn flatten_statement(&self, stmt: &Statement) -> Vec<(Span, FlatShape)> { - match stmt { - Statement::Expression(expr) => self.flatten_expression(expr), - Statement::Pipeline(pipeline) => self.flatten_pipeline(pipeline), - _ => vec![], - } - } - - pub fn flatten_expression(&self, expr: &Expression) -> Vec<(Span, FlatShape)> { - match &expr.expr { - Expr::BinaryOp(lhs, op, rhs) => { - let mut output = vec![]; - output.extend(self.flatten_expression(lhs)); - output.extend(self.flatten_expression(op)); - output.extend(self.flatten_expression(rhs)); - output - } - Expr::Block(block_id) => self.flatten_block(self.get_block(*block_id)), - Expr::Call(call) => { - let mut output = vec![(call.head, FlatShape::InternalCall)]; - for positional in &call.positional { - output.extend(self.flatten_expression(positional)); - } - output - } - Expr::ExternalCall(..) => { - vec![(expr.span, FlatShape::External)] - } - Expr::Garbage => { - vec![(expr.span, FlatShape::Garbage)] - } - Expr::Int(_) => { - vec![(expr.span, FlatShape::Int)] - } - Expr::Float(_) => { - vec![(expr.span, FlatShape::Float)] - } - Expr::Bool(_) => { - vec![(expr.span, FlatShape::Bool)] - } - - Expr::List(list) => { - let mut output = vec![]; - for l in list { - output.extend(self.flatten_expression(l)); - } - output - } - Expr::Keyword(_, span, expr) => { - let mut output = vec![(*span, FlatShape::Operator)]; - output.extend(self.flatten_expression(expr)); - output - } - Expr::Operator(_) => { - vec![(expr.span, FlatShape::Operator)] - } - Expr::Signature(_) => { - vec![(expr.span, FlatShape::Signature)] - } - Expr::String(_) => { - vec![(expr.span, FlatShape::String)] - } - Expr::Subexpression(block_id) => self.flatten_block(self.get_block(*block_id)), - Expr::Table(headers, cells) => { - let mut output = vec![]; - for e in headers { - output.extend(self.flatten_expression(e)); - } - for row in cells { - for expr in row { - output.extend(self.flatten_expression(expr)); - } - } - output - } - Expr::Var(_) => { - vec![(expr.span, FlatShape::Variable)] - } - } - } - - pub fn flatten_pipeline(&self, pipeline: &Pipeline) -> Vec<(Span, FlatShape)> { - let mut output = vec![]; - for expr in &pipeline.expressions { - output.extend(self.flatten_expression(expr)) - } - output +pub fn flatten_statement( + working_set: &StateWorkingSet, + stmt: &Statement, +) -> Vec<(Span, FlatShape)> { + match stmt { + Statement::Expression(expr) => flatten_expression(working_set, expr), + Statement::Pipeline(pipeline) => flatten_pipeline(working_set, pipeline), + _ => vec![], } } + +pub fn flatten_expression( + working_set: &StateWorkingSet, + expr: &Expression, +) -> Vec<(Span, FlatShape)> { + match &expr.expr { + Expr::BinaryOp(lhs, op, rhs) => { + let mut output = vec![]; + output.extend(flatten_expression(working_set, lhs)); + output.extend(flatten_expression(working_set, op)); + output.extend(flatten_expression(working_set, rhs)); + output + } + Expr::Block(block_id) => flatten_block(working_set, working_set.get_block(*block_id)), + Expr::Call(call) => { + let mut output = vec![(call.head, FlatShape::InternalCall)]; + for positional in &call.positional { + output.extend(flatten_expression(working_set, positional)); + } + output + } + Expr::ExternalCall(..) => { + vec![(expr.span, FlatShape::External)] + } + Expr::Garbage => { + vec![(expr.span, FlatShape::Garbage)] + } + Expr::Int(_) => { + vec![(expr.span, FlatShape::Int)] + } + Expr::Float(_) => { + vec![(expr.span, FlatShape::Float)] + } + Expr::Bool(_) => { + vec![(expr.span, FlatShape::Bool)] + } + + Expr::List(list) => { + let mut output = vec![]; + for l in list { + output.extend(flatten_expression(working_set, l)); + } + output + } + Expr::Keyword(_, span, expr) => { + let mut output = vec![(*span, FlatShape::Operator)]; + output.extend(flatten_expression(working_set, expr)); + output + } + Expr::Operator(_) => { + vec![(expr.span, FlatShape::Operator)] + } + Expr::Signature(_) => { + vec![(expr.span, FlatShape::Signature)] + } + Expr::String(_) => { + vec![(expr.span, FlatShape::String)] + } + Expr::Subexpression(block_id) => { + flatten_block(working_set, working_set.get_block(*block_id)) + } + Expr::Table(headers, cells) => { + let mut output = vec![]; + for e in headers { + output.extend(flatten_expression(working_set, e)); + } + for row in cells { + for expr in row { + output.extend(flatten_expression(working_set, expr)); + } + } + output + } + Expr::Var(_) => { + vec![(expr.span, FlatShape::Variable)] + } + } +} + +pub fn flatten_pipeline( + working_set: &StateWorkingSet, + pipeline: &Pipeline, +) -> Vec<(Span, FlatShape)> { + let mut output = vec![]; + for expr in &pipeline.expressions { + output.extend(flatten_expression(working_set, expr)) + } + output +} diff --git a/crates/nu-parser/src/lib.rs b/crates/nu-parser/src/lib.rs index 9649078b51..a15f2dfb42 100644 --- a/crates/nu-parser/src/lib.rs +++ b/crates/nu-parser/src/lib.rs @@ -3,12 +3,10 @@ mod flatten; mod lex; mod lite_parse; mod parser; -mod parser_state; mod type_check; pub use errors::ParseError; pub use flatten::FlatShape; pub use lex::{lex, Token, TokenContents}; pub use lite_parse::{lite_parse, LiteBlock}; -pub use parser::{Block, Call, Expr, Expression, Import, Operator, Pipeline, Statement, VarDecl}; -pub use parser_state::{ParserDelta, ParserState, ParserWorkingSet}; +pub use parser::{Import, VarDecl}; diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index e3e5c60936..0e1afd1ef4 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -1,268 +1,23 @@ -use std::{ - fmt::Display, - ops::{Index, IndexMut}, +use crate::{ + lex, lite_parse, + type_check::{math_result_type, type_compatible}, + LiteBlock, ParseError, Token, TokenContents, }; -use crate::{lex, lite_parse, LiteBlock, ParseError, ParserWorkingSet, Token, TokenContents}; - use nu_protocol::{ - span, BlockId, DeclId, Declaration, Flag, PositionalArg, Signature, Span, SyntaxShape, Type, - VarId, + span, Block, BlockId, Call, DeclId, Expr, Expression, Flag, Operator, Pipeline, PositionalArg, + Signature, Span, StateWorkingSet, Statement, SyntaxShape, Type, VarId, }; -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Operator { - Equal, - NotEqual, - LessThan, - GreaterThan, - LessThanOrEqual, - GreaterThanOrEqual, - Contains, - NotContains, - Plus, - Minus, - Multiply, - Divide, - In, - NotIn, - Modulo, - And, - Or, - Pow, -} - -impl Display for Operator { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Operator::Equal => write!(f, "=="), - Operator::NotEqual => write!(f, "!="), - Operator::LessThan => write!(f, "<"), - Operator::GreaterThan => write!(f, ">"), - Operator::Contains => write!(f, "=~"), - Operator::NotContains => write!(f, "!~"), - Operator::Plus => write!(f, "+"), - Operator::Minus => write!(f, "-"), - Operator::Multiply => write!(f, "*"), - Operator::Divide => write!(f, "/"), - Operator::In => write!(f, "in"), - Operator::NotIn => write!(f, "not-in"), - Operator::Modulo => write!(f, "mod"), - Operator::And => write!(f, "&&"), - Operator::Or => write!(f, "||"), - Operator::Pow => write!(f, "**"), - Operator::LessThanOrEqual => write!(f, "<="), - Operator::GreaterThanOrEqual => write!(f, ">="), - } - } -} - -#[derive(Debug, Clone)] -pub struct Call { - /// identifier of the declaration to call - pub decl_id: DeclId, - pub head: Span, - pub positional: Vec, - pub named: Vec<(String, Option)>, -} - -impl Default for Call { - fn default() -> Self { - Self::new() - } -} - -impl Call { - pub fn new() -> Call { - Self { - decl_id: 0, - head: Span::unknown(), - positional: vec![], - named: vec![], - } - } -} - -#[derive(Debug, Clone)] -pub enum Expr { - Bool(bool), - Int(i64), - Float(f64), - Var(VarId), - Call(Box), - ExternalCall(Vec, Vec>), - Operator(Operator), - BinaryOp(Box, Box, Box), //lhs, op, rhs - Subexpression(BlockId), - Block(BlockId), - List(Vec), - Table(Vec, Vec>), - Keyword(Vec, Span, Box), - String(String), // FIXME: improve this in the future? - Signature(Box), - Garbage, -} - -#[derive(Debug, Clone)] -pub struct Expression { - pub expr: Expr, - pub span: Span, - pub ty: Type, -} -impl Expression { - pub fn garbage(span: Span) -> Expression { - Expression { - expr: Expr::Garbage, - span, - ty: Type::Unknown, - } - } - pub fn precedence(&self) -> usize { - match &self.expr { - Expr::Operator(operator) => { - // Higher precedence binds tighter - - match operator { - Operator::Pow => 100, - Operator::Multiply | Operator::Divide | Operator::Modulo => 95, - Operator::Plus | Operator::Minus => 90, - Operator::NotContains - | Operator::Contains - | Operator::LessThan - | Operator::LessThanOrEqual - | Operator::GreaterThan - | Operator::GreaterThanOrEqual - | Operator::Equal - | Operator::NotEqual - | Operator::In - | Operator::NotIn => 80, - Operator::And => 50, - Operator::Or => 40, // TODO: should we have And and Or be different precedence? - } - } - _ => 0, - } - } - - pub fn as_block(&self) -> Option { - match self.expr { - Expr::Block(block_id) => Some(block_id), - _ => None, - } - } - - pub fn as_signature(&self) -> Option> { - match &self.expr { - Expr::Signature(sig) => Some(sig.clone()), - _ => None, - } - } - - pub fn as_list(&self) -> Option> { - match &self.expr { - Expr::List(list) => Some(list.clone()), - _ => None, - } - } - - pub fn as_keyword(&self) -> Option<&Expression> { - match &self.expr { - Expr::Keyword(_, _, expr) => Some(expr), - _ => None, - } - } - - pub fn as_var(&self) -> Option { - match self.expr { - Expr::Var(var_id) => Some(var_id), - _ => None, - } - } - - pub fn as_string(&self) -> Option { - match &self.expr { - Expr::String(string) => Some(string.clone()), - _ => None, - } - } -} - #[derive(Debug, Clone)] pub enum Import {} -#[derive(Debug, Clone)] -pub struct Block { - pub stmts: Vec, -} - -impl Block { - pub fn len(&self) -> usize { - self.stmts.len() - } - - pub fn is_empty(&self) -> bool { - self.stmts.is_empty() - } -} - -impl Index for Block { - type Output = Statement; - - fn index(&self, index: usize) -> &Self::Output { - &self.stmts[index] - } -} - -impl IndexMut for Block { - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - &mut self.stmts[index] - } -} - -impl Default for Block { - fn default() -> Self { - Self::new() - } -} - -impl Block { - pub fn new() -> Self { - Self { stmts: vec![] } - } -} - #[derive(Debug, Clone)] pub struct VarDecl { var_id: VarId, expression: Expression, } -#[derive(Debug, Clone)] -pub enum Statement { - Declaration(DeclId), - Pipeline(Pipeline), - Expression(Expression), -} - -#[derive(Debug, Clone)] -pub struct Pipeline { - pub expressions: Vec, -} - -impl Default for Pipeline { - fn default() -> Self { - Self::new() - } -} - -impl Pipeline { - pub fn new() -> Self { - Self { - expressions: vec![], - } - } -} - fn garbage(span: Span) -> Expression { Expression::garbage(span) } @@ -300,604 +55,558 @@ fn check_call(command: Span, sig: &Signature, call: &Call) -> Option } } -impl<'a> ParserWorkingSet<'a> { - pub fn parse_external_call(&mut self, spans: &[Span]) -> (Expression, Option) { - // TODO: add external parsing - let mut args = vec![]; - let name = self.get_span_contents(spans[0]).to_vec(); - for span in &spans[1..] { - args.push(self.get_span_contents(*span).to_vec()); - } - ( - Expression { - expr: Expr::ExternalCall(name, args), - span: span(spans), - ty: Type::Unknown, - }, - None, - ) +pub fn parse_external_call( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Expression, Option) { + // TODO: add external parsing + let mut args = vec![]; + let name = working_set.get_span_contents(spans[0]).to_vec(); + for span in &spans[1..] { + args.push(working_set.get_span_contents(*span).to_vec()); } + ( + Expression { + expr: Expr::ExternalCall(name, args), + span: span(spans), + ty: Type::Unknown, + }, + None, + ) +} - fn parse_long_flag( - &mut self, - spans: &[Span], - spans_idx: &mut usize, - sig: &Signature, - ) -> (Option, Option, Option) { - let arg_span = spans[*spans_idx]; - let arg_contents = self.get_span_contents(arg_span); +fn parse_long_flag( + working_set: &mut StateWorkingSet, + spans: &[Span], + spans_idx: &mut usize, + sig: &Signature, +) -> (Option, Option, Option) { + let arg_span = spans[*spans_idx]; + let arg_contents = working_set.get_span_contents(arg_span); - if arg_contents.starts_with(b"--") { - // FIXME: only use the first you find - let split: Vec<_> = arg_contents.split(|x| *x == b'=').collect(); - let long_name = String::from_utf8(split[0].into()); - if let Ok(long_name) = long_name { - if let Some(flag) = sig.get_long_flag(&long_name) { - if let Some(arg_shape) = &flag.arg { - if split.len() > 1 { - // and we also have the argument - let mut span = arg_span; - span.start += long_name.len() + 1; //offset by long flag and '=' - let (arg, err) = self.parse_value(span, arg_shape); + if arg_contents.starts_with(b"--") { + // FIXME: only use the first you find + let split: Vec<_> = arg_contents.split(|x| *x == b'=').collect(); + let long_name = String::from_utf8(split[0].into()); + if let Ok(long_name) = long_name { + if let Some(flag) = sig.get_long_flag(&long_name) { + if let Some(arg_shape) = &flag.arg { + if split.len() > 1 { + // and we also have the argument + let mut span = arg_span; + span.start += long_name.len() + 1; //offset by long flag and '=' + let (arg, err) = parse_value(working_set, span, arg_shape); - (Some(long_name), Some(arg), err) - } else if let Some(arg) = spans.get(*spans_idx + 1) { - let (arg, err) = self.parse_value(*arg, arg_shape); + (Some(long_name), Some(arg), err) + } else if let Some(arg) = spans.get(*spans_idx + 1) { + let (arg, err) = parse_value(working_set, *arg, arg_shape); - *spans_idx += 1; - (Some(long_name), Some(arg), err) - } else { - ( - Some(long_name), - None, - Some(ParseError::MissingFlagParam(arg_span)), - ) - } + *spans_idx += 1; + (Some(long_name), Some(arg), err) } else { - // A flag with no argument - (Some(long_name), None, None) + ( + Some(long_name), + None, + Some(ParseError::MissingFlagParam(arg_span)), + ) } } else { - ( - Some(long_name), - None, - Some(ParseError::UnknownFlag(arg_span)), - ) + // A flag with no argument + (Some(long_name), None, None) } } else { - (Some("--".into()), None, Some(ParseError::NonUtf8(arg_span))) + ( + Some(long_name), + None, + Some(ParseError::UnknownFlag(arg_span)), + ) } } else { - (None, None, None) + (Some("--".into()), None, Some(ParseError::NonUtf8(arg_span))) } + } else { + (None, None, None) } +} - fn parse_short_flags( - &mut self, - spans: &[Span], - spans_idx: &mut usize, - positional_idx: usize, - sig: &Signature, - ) -> (Option>, Option) { - let mut error = None; - let arg_span = spans[*spans_idx]; +fn parse_short_flags( + working_set: &mut StateWorkingSet, + spans: &[Span], + spans_idx: &mut usize, + positional_idx: usize, + sig: &Signature, +) -> (Option>, Option) { + let mut error = None; + let arg_span = spans[*spans_idx]; - let arg_contents = self.get_span_contents(arg_span); + let arg_contents = working_set.get_span_contents(arg_span); - if arg_contents.starts_with(b"-") && arg_contents.len() > 1 { - let short_flags = &arg_contents[1..]; - let mut found_short_flags = vec![]; - let mut unmatched_short_flags = vec![]; - for short_flag in short_flags.iter().enumerate() { - let short_flag_char = char::from(*short_flag.1); - let orig = arg_span; - let short_flag_span = Span { - start: orig.start + 1 + short_flag.0, - end: orig.start + 1 + short_flag.0 + 1, - }; - if let Some(flag) = sig.get_short_flag(short_flag_char) { - // If we require an arg and are in a batch of short flags, error - if !found_short_flags.is_empty() && flag.arg.is_some() { - error = - error.or(Some(ParseError::ShortFlagBatchCantTakeArg(short_flag_span))) - } - found_short_flags.push(flag); - } else { - unmatched_short_flags.push(short_flag_span); + if arg_contents.starts_with(b"-") && arg_contents.len() > 1 { + let short_flags = &arg_contents[1..]; + let mut found_short_flags = vec![]; + let mut unmatched_short_flags = vec![]; + for short_flag in short_flags.iter().enumerate() { + let short_flag_char = char::from(*short_flag.1); + let orig = arg_span; + let short_flag_span = Span { + start: orig.start + 1 + short_flag.0, + end: orig.start + 1 + short_flag.0 + 1, + }; + if let Some(flag) = sig.get_short_flag(short_flag_char) { + // If we require an arg and are in a batch of short flags, error + if !found_short_flags.is_empty() && flag.arg.is_some() { + error = error.or(Some(ParseError::ShortFlagBatchCantTakeArg(short_flag_span))) } + found_short_flags.push(flag); + } else { + unmatched_short_flags.push(short_flag_span); } + } - if found_short_flags.is_empty() { - // check to see if we have a negative number - if let Some(positional) = sig.get_positional(positional_idx) { - if positional.shape == SyntaxShape::Int - || positional.shape == SyntaxShape::Number - { - if String::from_utf8_lossy(arg_contents).parse::().is_ok() { - return (None, None); - } else if let Some(first) = unmatched_short_flags.first() { - error = error.or(Some(ParseError::UnknownFlag(*first))); - } + if found_short_flags.is_empty() { + // check to see if we have a negative number + if let Some(positional) = sig.get_positional(positional_idx) { + if positional.shape == SyntaxShape::Int || positional.shape == SyntaxShape::Number { + if String::from_utf8_lossy(arg_contents).parse::().is_ok() { + return (None, None); } else if let Some(first) = unmatched_short_flags.first() { error = error.or(Some(ParseError::UnknownFlag(*first))); } } else if let Some(first) = unmatched_short_flags.first() { error = error.or(Some(ParseError::UnknownFlag(*first))); } - } else if !unmatched_short_flags.is_empty() { - if let Some(first) = unmatched_short_flags.first() { - error = error.or(Some(ParseError::UnknownFlag(*first))); - } + } else if let Some(first) = unmatched_short_flags.first() { + error = error.or(Some(ParseError::UnknownFlag(*first))); + } + } else if !unmatched_short_flags.is_empty() { + if let Some(first) = unmatched_short_flags.first() { + error = error.or(Some(ParseError::UnknownFlag(*first))); } - - (Some(found_short_flags), error) - } else { - (None, None) } + + (Some(found_short_flags), error) + } else { + (None, None) } +} - fn first_kw_idx( - &self, - decl: &Declaration, - spans: &[Span], - spans_idx: usize, - positional_idx: usize, - ) -> (Option, usize) { - for idx in (positional_idx + 1)..decl.signature.num_positionals() { - if let Some(PositionalArg { - shape: SyntaxShape::Keyword(kw, ..), - .. - }) = decl.signature.get_positional(idx) - { - #[allow(clippy::needless_range_loop)] - for span_idx in spans_idx..spans.len() { - let contents = self.get_span_contents(spans[span_idx]); +fn first_kw_idx( + working_set: &StateWorkingSet, + signature: &Signature, + spans: &[Span], + spans_idx: usize, + positional_idx: usize, +) -> (Option, usize) { + for idx in (positional_idx + 1)..signature.num_positionals() { + if let Some(PositionalArg { + shape: SyntaxShape::Keyword(kw, ..), + .. + }) = signature.get_positional(idx) + { + #[allow(clippy::needless_range_loop)] + for span_idx in spans_idx..spans.len() { + let contents = working_set.get_span_contents(spans[span_idx]); - if contents == kw { - return (Some(idx), span_idx); - } + if contents == kw { + return (Some(idx), span_idx); } } } - (None, spans.len()) } + (None, spans.len()) +} - fn calculate_end_span( - &self, - decl: &Declaration, - spans: &[Span], - spans_idx: usize, - positional_idx: usize, - ) -> usize { - if decl.signature.rest_positional.is_some() { - spans.len() - } else { - let (kw_pos, kw_idx) = self.first_kw_idx(decl, spans, spans_idx, positional_idx); +fn calculate_end_span( + working_set: &StateWorkingSet, + signature: &Signature, + spans: &[Span], + spans_idx: usize, + positional_idx: usize, +) -> usize { + if signature.rest_positional.is_some() { + spans.len() + } else { + let (kw_pos, kw_idx) = + first_kw_idx(working_set, signature, spans, spans_idx, positional_idx); - if let Some(kw_pos) = kw_pos { - // We found a keyword. Keywords, once found, create a guidepost to - // show us where the positionals will lay into the arguments. Because they're - // keywords, they get to set this by being present + if let Some(kw_pos) = kw_pos { + // We found a keyword. Keywords, once found, create a guidepost to + // show us where the positionals will lay into the arguments. Because they're + // keywords, they get to set this by being present - let positionals_between = kw_pos - positional_idx - 1; - if positionals_between > (kw_idx - spans_idx) { - kw_idx - } else { - kw_idx - positionals_between - } + let positionals_between = kw_pos - positional_idx - 1; + if positionals_between > (kw_idx - spans_idx) { + kw_idx } else { - // Make space for the remaining require positionals, if we can - if positional_idx < decl.signature.required_positional.len() - && spans.len() > (decl.signature.required_positional.len() - positional_idx) - { - spans.len() - (decl.signature.required_positional.len() - positional_idx - 1) + kw_idx - positionals_between + } + } else { + // Make space for the remaining require positionals, if we can + if positional_idx < signature.required_positional.len() + && spans.len() > (signature.required_positional.len() - positional_idx) + { + spans.len() - (signature.required_positional.len() - positional_idx - 1) + } else { + if signature.num_positionals_after(positional_idx) == 0 { + spans.len() } else { - if decl.signature.num_positionals_after(positional_idx) == 0 { - spans.len() - } else { - spans_idx + 1 - } + spans_idx + 1 } } } } +} - fn parse_multispan_value( - &mut self, - spans: &[Span], - spans_idx: &mut usize, - shape: &SyntaxShape, - ) -> (Expression, Option) { - let mut error = None; +fn parse_multispan_value( + working_set: &mut StateWorkingSet, + spans: &[Span], + spans_idx: &mut usize, + shape: &SyntaxShape, +) -> (Expression, Option) { + let mut error = None; - match shape { - SyntaxShape::VarWithOptType => { - let (arg, err) = self.parse_var_with_opt_type(spans, spans_idx); - error = error.or(err); + match shape { + SyntaxShape::VarWithOptType => { + let (arg, err) = parse_var_with_opt_type(working_set, spans, spans_idx); + error = error.or(err); - (arg, error) + (arg, error) + } + SyntaxShape::RowCondition => { + let (arg, err) = parse_row_condition(working_set, &spans[*spans_idx..]); + error = error.or(err); + *spans_idx = spans.len() - 1; + + (arg, error) + } + SyntaxShape::Expression => { + let (arg, err) = parse_expression(working_set, &spans[*spans_idx..]); + error = error.or(err); + *spans_idx = spans.len() - 1; + + (arg, error) + } + SyntaxShape::Keyword(keyword, arg) => { + let arg_span = spans[*spans_idx]; + + let arg_contents = working_set.get_span_contents(arg_span); + + if arg_contents != keyword { + // When keywords mismatch, this is a strong indicator of something going wrong. + // We won't often override the current error, but as this is a strong indicator + // go ahead and override the current error and tell the user about the missing + // keyword/literal. + error = Some(ParseError::ExpectedKeyword( + String::from_utf8_lossy(keyword).into(), + arg_span, + )) } - SyntaxShape::RowCondition => { - let (arg, err) = self.parse_row_condition(&spans[*spans_idx..]); - error = error.or(err); - *spans_idx = spans.len() - 1; - (arg, error) - } - SyntaxShape::Expression => { - let (arg, err) = self.parse_expression(&spans[*spans_idx..]); - error = error.or(err); - *spans_idx = spans.len() - 1; - - (arg, error) - } - SyntaxShape::Keyword(keyword, arg) => { - let arg_span = spans[*spans_idx]; - - let arg_contents = self.get_span_contents(arg_span); - - if arg_contents != keyword { - // When keywords mismatch, this is a strong indicator of something going wrong. - // We won't often override the current error, but as this is a strong indicator - // go ahead and override the current error and tell the user about the missing - // keyword/literal. - error = Some(ParseError::ExpectedKeyword( + *spans_idx += 1; + if *spans_idx >= spans.len() { + error = error.or_else(|| { + Some(ParseError::KeywordMissingArgument( String::from_utf8_lossy(keyword).into(), - arg_span, + spans[*spans_idx - 1], )) - } - - *spans_idx += 1; - if *spans_idx >= spans.len() { - error = error.or_else(|| { - Some(ParseError::KeywordMissingArgument( - String::from_utf8_lossy(keyword).into(), - spans[*spans_idx - 1], - )) - }); - return ( - Expression { - expr: Expr::Keyword( - keyword.clone(), - spans[*spans_idx - 1], - Box::new(Expression::garbage(arg_span)), - ), - span: arg_span, - ty: Type::Unknown, - }, - error, - ); - } - let keyword_span = spans[*spans_idx - 1]; - let (expr, err) = self.parse_multispan_value(spans, spans_idx, arg); - error = error.or(err); - let ty = expr.ty.clone(); - - ( + }); + return ( Expression { - expr: Expr::Keyword(keyword.clone(), keyword_span, Box::new(expr)), + expr: Expr::Keyword( + keyword.clone(), + spans[*spans_idx - 1], + Box::new(Expression::garbage(arg_span)), + ), span: arg_span, - ty, + ty: Type::Unknown, }, error, - ) + ); } - _ => { - // All other cases are single-span values - let arg_span = spans[*spans_idx]; + let keyword_span = spans[*spans_idx - 1]; + let (expr, err) = parse_multispan_value(working_set, spans, spans_idx, arg); + error = error.or(err); + let ty = expr.ty.clone(); - let (arg, err) = self.parse_value(arg_span, shape); - error = error.or(err); + ( + Expression { + expr: Expr::Keyword(keyword.clone(), keyword_span, Box::new(expr)), + span: arg_span, + ty, + }, + error, + ) + } + _ => { + // All other cases are single-span values + let arg_span = spans[*spans_idx]; - (arg, error) + let (arg, err) = parse_value(working_set, arg_span, shape); + error = error.or(err); + + (arg, error) + } + } +} + +pub fn parse_internal_call( + working_set: &mut StateWorkingSet, + command_span: Span, + spans: &[Span], + decl_id: usize, +) -> (Box, Span, Option) { + let mut error = None; + + let mut call = Call::new(); + call.decl_id = decl_id; + call.head = command_span; + + let signature = working_set.get_decl(decl_id).signature(); + + // The index into the positional parameter in the definition + let mut positional_idx = 0; + + // The index into the spans of argument data given to parse + // Starting at the first argument + let mut spans_idx = 0; + + while spans_idx < spans.len() { + let arg_span = spans[spans_idx]; + + // Check if we're on a long flag, if so, parse + let (long_name, arg, err) = parse_long_flag(working_set, spans, &mut spans_idx, &signature); + if let Some(long_name) = long_name { + // We found a long flag, like --bar + error = error.or(err); + call.named.push((long_name, arg)); + spans_idx += 1; + continue; + } + + // Check if we're on a short flag or group of short flags, if so, parse + let (short_flags, err) = parse_short_flags( + working_set, + spans, + &mut spans_idx, + positional_idx, + &signature, + ); + + if let Some(short_flags) = short_flags { + error = error.or(err); + for flag in short_flags { + if let Some(arg_shape) = flag.arg { + if let Some(arg) = spans.get(spans_idx + 1) { + let (arg, err) = parse_value(working_set, *arg, &arg_shape); + error = error.or(err); + + call.named.push((flag.long.clone(), Some(arg))); + spans_idx += 1; + } else { + error = error.or(Some(ParseError::MissingFlagParam(arg_span))) + } + } else { + call.named.push((flag.long.clone(), None)); + } } + spans_idx += 1; + continue; + } + + // Parse a positional arg if there is one + if let Some(positional) = signature.get_positional(positional_idx) { + //Make sure we leave enough spans for the remaining positionals + let decl = working_set.get_decl(decl_id); + + let end = calculate_end_span(working_set, &signature, spans, spans_idx, positional_idx); + + // println!( + // "start: {} end: {} positional_idx: {}", + // spans_idx, end, positional_idx + // ); + + let orig_idx = spans_idx; + let (arg, err) = parse_multispan_value( + working_set, + &spans[..end], + &mut spans_idx, + &positional.shape, + ); + error = error.or(err); + + let arg = if !type_compatible(&positional.shape.to_type(), &arg.ty) { + let span = span(&spans[orig_idx..spans_idx]); + error = error.or_else(|| { + Some(ParseError::TypeMismatch( + positional.shape.to_type(), + arg.ty, + arg.span, + )) + }); + Expression::garbage(span) + } else { + arg + }; + call.positional.push(arg); + positional_idx += 1; + } else { + call.positional.push(Expression::garbage(arg_span)); + error = error.or(Some(ParseError::ExtraPositional(arg_span))) + } + + error = error.or(err); + spans_idx += 1; + } + + let err = check_call(command_span, &signature, &call); + error = error.or(err); + + // FIXME: type unknown + (Box::new(call), span(spans), error) +} + +pub fn parse_call( + working_set: &mut StateWorkingSet, + spans: &[Span], + expand_aliases: bool, +) -> (Expression, Option) { + // assume spans.len() > 0? + let mut pos = 0; + let mut shorthand = vec![]; + + while pos < spans.len() { + // Check if there is any environment shorthand + let name = working_set.get_span_contents(spans[pos]); + let split: Vec<_> = name.splitn(2, |x| *x == b'=').collect(); + if split.len() == 2 { + shorthand.push(split); + pos += 1; + } else { + break; } } - pub fn parse_internal_call( - &mut self, - command_span: Span, - spans: &[Span], - decl_id: usize, - ) -> (Box, Span, Option) { - let mut error = None; + if pos == spans.len() { + return ( + Expression::garbage(span(spans)), + Some(ParseError::UnknownCommand(spans[0])), + ); + } - let mut call = Call::new(); - call.decl_id = decl_id; - call.head = command_span; + let name = working_set.get_span_contents(spans[pos]); - let signature = self.get_decl(decl_id).signature.clone(); + let cmd_start = pos; - // The index into the positional parameter in the definition - let mut positional_idx = 0; - - // The index into the spans of argument data given to parse - // Starting at the first argument - let mut spans_idx = 0; - - while spans_idx < spans.len() { - let arg_span = spans[spans_idx]; - - // Check if we're on a long flag, if so, parse - let (long_name, arg, err) = self.parse_long_flag(spans, &mut spans_idx, &signature); - if let Some(long_name) = long_name { - // We found a long flag, like --bar - error = error.or(err); - call.named.push((long_name, arg)); - spans_idx += 1; - continue; + if expand_aliases { + if let Some(expansion) = working_set.find_alias(&name) { + let orig_span = spans[pos]; + //let mut spans = spans.to_vec(); + let mut new_spans: Vec = vec![]; + new_spans.extend(&spans[0..pos]); + new_spans.extend(expansion); + if spans.len() > pos { + new_spans.extend(&spans[(pos + 1)..]); } - // Check if we're on a short flag or group of short flags, if so, parse - let (short_flags, err) = - self.parse_short_flags(spans, &mut spans_idx, positional_idx, &signature); + let (result, err) = parse_call(working_set, &new_spans, false); - if let Some(short_flags) = short_flags { - error = error.or(err); - for flag in short_flags { - if let Some(arg_shape) = flag.arg { - if let Some(arg) = spans.get(spans_idx + 1) { - let (arg, err) = self.parse_value(*arg, &arg_shape); - error = error.or(err); - - call.named.push((flag.long.clone(), Some(arg))); - spans_idx += 1; - } else { - error = error.or(Some(ParseError::MissingFlagParam(arg_span))) - } - } else { - call.named.push((flag.long.clone(), None)); + let expression = match result { + Expression { + expr: Expr::Call(mut call), + span, + ty, + } => { + call.head = orig_span; + Expression { + expr: Expr::Call(call), + span, + ty, } } - spans_idx += 1; - continue; - } + x => x, + }; - // Parse a positional arg if there is one - if let Some(positional) = signature.get_positional(positional_idx) { - //Make sure we leave enough spans for the remaining positionals - let decl = self.get_decl(decl_id); - - let end = self.calculate_end_span(&decl, spans, spans_idx, positional_idx); - - // println!( - // "start: {} end: {} positional_idx: {}", - // spans_idx, end, positional_idx - // ); - - let orig_idx = spans_idx; - let (arg, err) = - self.parse_multispan_value(&spans[..end], &mut spans_idx, &positional.shape); - error = error.or(err); - - let arg = if !Self::type_compatible(&positional.shape.to_type(), &arg.ty) { - let span = span(&spans[orig_idx..spans_idx]); - error = error.or_else(|| { - Some(ParseError::TypeMismatch( - positional.shape.to_type(), - arg.ty, - arg.span, - )) - }); - Expression::garbage(span) - } else { - arg - }; - call.positional.push(arg); - positional_idx += 1; - } else { - call.positional.push(Expression::garbage(arg_span)); - error = error.or(Some(ParseError::ExtraPositional(arg_span))) - } - - error = error.or(err); - spans_idx += 1; + return (expression, err); } - - let err = check_call(command_span, &signature, &call); - error = error.or(err); - - // FIXME: type unknown - (Box::new(call), span(spans), error) } - pub fn parse_call( - &mut self, - spans: &[Span], - expand_aliases: bool, - ) -> (Expression, Option) { - // assume spans.len() > 0? - let mut pos = 0; - let mut shorthand = vec![]; + pos += 1; + if let Some(mut decl_id) = working_set.find_decl(name) { + let mut name = name.to_vec(); while pos < spans.len() { - // Check if there is any environment shorthand - let name = self.get_span_contents(spans[pos]); - let split: Vec<_> = name.splitn(2, |x| *x == b'=').collect(); - if split.len() == 2 { - shorthand.push(split); - pos += 1; + // look to see if it's a subcommand + let mut new_name = name.to_vec(); + new_name.push(b' '); + new_name.extend(working_set.get_span_contents(spans[pos])); + + if expand_aliases { + if let Some(expansion) = working_set.find_alias(&new_name) { + let orig_span = span(&spans[cmd_start..pos + 1]); + //let mut spans = spans.to_vec(); + let mut new_spans: Vec = vec![]; + new_spans.extend(&spans[0..cmd_start]); + new_spans.extend(expansion); + if spans.len() > pos { + new_spans.extend(&spans[(pos + 1)..]); + } + + let (result, err) = parse_call(working_set, &new_spans, false); + + let expression = match result { + Expression { + expr: Expr::Call(mut call), + span, + ty, + } => { + call.head = orig_span; + Expression { + expr: Expr::Call(call), + span, + ty, + } + } + x => x, + }; + + return (expression, err); + } + } + + if let Some(did) = working_set.find_decl(&new_name) { + decl_id = did; } else { break; } + name = new_name; + pos += 1; } - - if pos == spans.len() { - return ( - Expression::garbage(span(spans)), - Some(ParseError::UnknownCommand(spans[0])), - ); - } - - let name = self.get_span_contents(spans[pos]); - - let cmd_start = pos; - - if expand_aliases { - if let Some(expansion) = self.find_alias(&name) { - let orig_span = spans[pos]; - //let mut spans = spans.to_vec(); - let mut new_spans: Vec = vec![]; - new_spans.extend(&spans[0..pos]); - new_spans.extend(expansion); - if spans.len() > pos { - new_spans.extend(&spans[(pos + 1)..]); - } - - let (result, err) = self.parse_call(&new_spans, false); - - let expression = match result { - Expression { - expr: Expr::Call(mut call), - span, - ty, - } => { - call.head = orig_span; - Expression { - expr: Expr::Call(call), - span, - ty, - } - } - x => x, - }; - - return (expression, err); - } - } - - pos += 1; - - if let Some(mut decl_id) = self.find_decl(name) { - let mut name = name.to_vec(); - while pos < spans.len() { - // look to see if it's a subcommand - let mut new_name = name.to_vec(); - new_name.push(b' '); - new_name.extend(self.get_span_contents(spans[pos])); - - if expand_aliases { - if let Some(expansion) = self.find_alias(&new_name) { - let orig_span = span(&spans[cmd_start..pos + 1]); - //let mut spans = spans.to_vec(); - let mut new_spans: Vec = vec![]; - new_spans.extend(&spans[0..cmd_start]); - new_spans.extend(expansion); - if spans.len() > pos { - new_spans.extend(&spans[(pos + 1)..]); - } - - let (result, err) = self.parse_call(&new_spans, false); - - let expression = match result { - Expression { - expr: Expr::Call(mut call), - span, - ty, - } => { - call.head = orig_span; - Expression { - expr: Expr::Call(call), - span, - ty, - } - } - x => x, - }; - - return (expression, err); - } - } - - if let Some(did) = self.find_decl(&new_name) { - decl_id = did; - } else { - break; - } - name = new_name; - pos += 1; - } - // parse internal command - let (call, _, err) = - self.parse_internal_call(span(&spans[0..pos]), &spans[pos..], decl_id); - ( - Expression { - expr: Expr::Call(call), - span: span(spans), - ty: Type::Unknown, // FIXME - }, - err, - ) - } else { - self.parse_external_call(spans) - } + // parse internal command + let (call, _, err) = + parse_internal_call(working_set, span(&spans[0..pos]), &spans[pos..], decl_id); + ( + Expression { + expr: Expr::Call(call), + span: span(spans), + ty: Type::Unknown, // FIXME + }, + err, + ) + } else { + parse_external_call(working_set, spans) } +} - pub fn parse_int(&mut self, token: &str, span: Span) -> (Expression, Option) { - if let Some(token) = token.strip_prefix("0x") { - if let Ok(v) = i64::from_str_radix(token, 16) { - ( - Expression { - expr: Expr::Int(v), - span, - ty: Type::Int, - }, - None, - ) - } else { - ( - garbage(span), - Some(ParseError::Mismatch( - "int".into(), - "incompatible int".into(), - span, - )), - ) - } - } else if let Some(token) = token.strip_prefix("0b") { - if let Ok(v) = i64::from_str_radix(token, 2) { - ( - Expression { - expr: Expr::Int(v), - span, - ty: Type::Int, - }, - None, - ) - } else { - ( - garbage(span), - Some(ParseError::Mismatch( - "int".into(), - "incompatible int".into(), - span, - )), - ) - } - } else if let Some(token) = token.strip_prefix("0o") { - if let Ok(v) = i64::from_str_radix(token, 8) { - ( - Expression { - expr: Expr::Int(v), - span, - ty: Type::Int, - }, - None, - ) - } else { - ( - garbage(span), - Some(ParseError::Mismatch( - "int".into(), - "incompatible int".into(), - span, - )), - ) - } - } else if let Ok(x) = token.parse::() { +pub fn parse_int( + working_set: &mut StateWorkingSet, + token: &str, + span: Span, +) -> (Expression, Option) { + if let Some(token) = token.strip_prefix("0x") { + if let Ok(v) = i64::from_str_radix(token, 16) { ( Expression { - expr: Expr::Int(x), + expr: Expr::Int(v), span, ty: Type::Int, }, @@ -906,1427 +615,1477 @@ impl<'a> ParserWorkingSet<'a> { } else { ( garbage(span), - Some(ParseError::Expected("int".into(), span)), + Some(ParseError::Mismatch( + "int".into(), + "incompatible int".into(), + span, + )), ) } - } - - pub fn parse_float(&mut self, token: &str, span: Span) -> (Expression, Option) { - if let Ok(x) = token.parse::() { + } else if let Some(token) = token.strip_prefix("0b") { + if let Ok(v) = i64::from_str_radix(token, 2) { ( Expression { - expr: Expr::Float(x), + expr: Expr::Int(v), span, - ty: Type::Float, + ty: Type::Int, }, None, ) } else { ( garbage(span), - Some(ParseError::Expected("float".into(), span)), + Some(ParseError::Mismatch( + "int".into(), + "incompatible int".into(), + span, + )), ) } - } - - pub fn parse_number(&mut self, token: &str, span: Span) -> (Expression, Option) { - if let (x, None) = self.parse_int(token, span) { - (x, None) - } else if let (x, None) = self.parse_float(token, span) { - (x, None) + } else if let Some(token) = token.strip_prefix("0o") { + if let Ok(v) = i64::from_str_radix(token, 8) { + ( + Expression { + expr: Expr::Int(v), + span, + ty: Type::Int, + }, + None, + ) } else { ( garbage(span), - Some(ParseError::Expected("number".into(), span)), + Some(ParseError::Mismatch( + "int".into(), + "incompatible int".into(), + span, + )), ) } - } - - pub(crate) fn parse_dollar_expr(&mut self, span: Span) -> (Expression, Option) { - let contents = self.get_span_contents(span); - - if contents.starts_with(b"$\"") { - self.parse_string_interpolation(span) - } else { - self.parse_variable_expr(span) - } - } - - pub fn parse_string_interpolation(&mut self, span: Span) -> (Expression, Option) { - #[derive(PartialEq, Eq, Debug)] - enum InterpolationMode { - String, - Expression, - } - let mut error = None; - - let contents = self.get_span_contents(span); - - let start = if contents.starts_with(b"$\"") { - span.start + 2 - } else { - span.start - }; - - let end = if contents.ends_with(b"\"") && contents.len() > 2 { - span.end - 1 - } else { - span.end - }; - - let inner_span = Span { start, end }; - let contents = self.get_span_contents(inner_span).to_vec(); - - let mut output = vec![]; - let mut mode = InterpolationMode::String; - let mut token_start = start; - let mut depth = 0; - - let mut b = start; - - #[allow(clippy::needless_range_loop)] - while b != end { - if contents[b - start] == b'(' && mode == InterpolationMode::String { - depth = 1; - mode = InterpolationMode::Expression; - if token_start < b { - let span = Span { - start: token_start, - end: b, - }; - let str_contents = self.get_span_contents(span); - output.push(Expression { - expr: Expr::String(String::from_utf8_lossy(str_contents).to_string()), - span, - ty: Type::String, - }); - } - token_start = b; - } else if contents[b - start] == b'(' && mode == InterpolationMode::Expression { - depth += 1; - } else if contents[b - start] == b')' && mode == InterpolationMode::Expression { - match depth { - 0 => {} - 1 => { - mode = InterpolationMode::String; - - if token_start < b { - let span = Span { - start: token_start, - end: b + 1, - }; - - let (expr, err) = self.parse_full_column_path(span); - error = error.or(err); - output.push(expr); - } - - token_start = b + 1; - } - _ => depth -= 1, - } - } - b += 1; - } - - match mode { - InterpolationMode::String => { - if token_start < end { - let span = Span { - start: token_start, - end, - }; - let str_contents = self.get_span_contents(span); - output.push(Expression { - expr: Expr::String(String::from_utf8_lossy(str_contents).to_string()), - span, - ty: Type::String, - }); - } - } - InterpolationMode::Expression => { - if token_start < end { - let span = Span { - start: token_start, - end, - }; - - let (expr, err) = self.parse_full_column_path(span); - error = error.or(err); - output.push(expr); - } - } - } - - if let Some(decl_id) = self.find_decl(b"build-string") { - ( - Expression { - expr: Expr::Call(Box::new(Call { - head: Span { - start: span.start, - end: span.start + 2, - }, - named: vec![], - positional: output, - decl_id, - })), - span, - ty: Type::String, - }, - error, - ) - } else { - ( - Expression::garbage(span), - Some(ParseError::UnknownCommand(span)), - ) - } - } - - pub fn parse_variable_expr(&mut self, span: Span) -> (Expression, Option) { - let contents = self.get_span_contents(span); - - if contents == b"$true" { - return ( - Expression { - expr: Expr::Bool(true), - span, - ty: Type::Bool, - }, - None, - ); - } else if contents == b"$false" { - return ( - Expression { - expr: Expr::Bool(false), - span, - ty: Type::Bool, - }, - None, - ); - } - - let (id, err) = self.parse_variable(span); - - if err.is_none() { - if let Some(id) = id { - ( - Expression { - expr: Expr::Var(id), - span, - ty: self.get_variable(id).clone(), - }, - None, - ) - } else { - let name = self.get_span_contents(span).to_vec(); - // this seems okay to set it to unknown here, but we should double-check - let id = self.add_variable(name, Type::Unknown); - ( - Expression { - expr: Expr::Var(id), - span, - ty: Type::Unknown, - }, - None, - ) - } - } else { - (garbage(span), err) - } - } - - pub fn parse_full_column_path(&mut self, span: Span) -> (Expression, Option) { - // FIXME: assume for now a paren expr, but needs more - let bytes = self.get_span_contents(span); - let mut error = None; - - let mut start = span.start; - let mut end = span.end; - - if bytes.starts_with(b"(") { - start += 1; - } - if bytes.ends_with(b")") { - end -= 1; - } else { - error = error.or_else(|| { - Some(ParseError::Unclosed( - ")".into(), - Span { - start: end, - end: end + 1, - }, - )) - }); - } - - let span = Span { start, end }; - - let source = self.get_span_contents(span); - - let (output, err) = lex(source, start, &[], &[]); - error = error.or(err); - - let (output, err) = lite_parse(&output); - error = error.or(err); - - let (output, err) = self.parse_block(&output, true); - error = error.or(err); - - let block_id = self.add_block(output); - + } else if let Ok(x) = token.parse::() { ( Expression { - expr: Expr::Subexpression(block_id), + expr: Expr::Int(x), span, - ty: Type::Unknown, // FIXME + ty: Type::Int, + }, + None, + ) + } else { + ( + garbage(span), + Some(ParseError::Expected("int".into(), span)), + ) + } +} + +pub fn parse_float( + working_set: &mut StateWorkingSet, + token: &str, + span: Span, +) -> (Expression, Option) { + if let Ok(x) = token.parse::() { + ( + Expression { + expr: Expr::Float(x), + span, + ty: Type::Float, + }, + None, + ) + } else { + ( + garbage(span), + Some(ParseError::Expected("float".into(), span)), + ) + } +} + +pub fn parse_number( + working_set: &mut StateWorkingSet, + token: &str, + span: Span, +) -> (Expression, Option) { + if let (x, None) = parse_int(working_set, token, span) { + (x, None) + } else if let (x, None) = parse_float(working_set, token, span) { + (x, None) + } else { + ( + garbage(span), + Some(ParseError::Expected("number".into(), span)), + ) + } +} + +pub(crate) fn parse_dollar_expr( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + let contents = working_set.get_span_contents(span); + + if contents.starts_with(b"$\"") { + parse_string_interpolation(working_set, span) + } else { + parse_variable_expr(working_set, span) + } +} + +pub fn parse_string_interpolation( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + #[derive(PartialEq, Eq, Debug)] + enum InterpolationMode { + String, + Expression, + } + let mut error = None; + + let contents = working_set.get_span_contents(span); + + let start = if contents.starts_with(b"$\"") { + span.start + 2 + } else { + span.start + }; + + let end = if contents.ends_with(b"\"") && contents.len() > 2 { + span.end - 1 + } else { + span.end + }; + + let inner_span = Span { start, end }; + let contents = working_set.get_span_contents(inner_span).to_vec(); + + let mut output = vec![]; + let mut mode = InterpolationMode::String; + let mut token_start = start; + let mut depth = 0; + + let mut b = start; + + #[allow(clippy::needless_range_loop)] + while b != end { + if contents[b - start] == b'(' && mode == InterpolationMode::String { + depth = 1; + mode = InterpolationMode::Expression; + if token_start < b { + let span = Span { + start: token_start, + end: b, + }; + let str_contents = working_set.get_span_contents(span); + output.push(Expression { + expr: Expr::String(String::from_utf8_lossy(str_contents).to_string()), + span, + ty: Type::String, + }); + } + token_start = b; + } else if contents[b - start] == b'(' && mode == InterpolationMode::Expression { + depth += 1; + } else if contents[b - start] == b')' && mode == InterpolationMode::Expression { + match depth { + 0 => {} + 1 => { + mode = InterpolationMode::String; + + if token_start < b { + let span = Span { + start: token_start, + end: b + 1, + }; + + let (expr, err) = parse_full_column_path(working_set, span); + error = error.or(err); + output.push(expr); + } + + token_start = b + 1; + } + _ => depth -= 1, + } + } + b += 1; + } + + match mode { + InterpolationMode::String => { + if token_start < end { + let span = Span { + start: token_start, + end, + }; + let str_contents = working_set.get_span_contents(span); + output.push(Expression { + expr: Expr::String(String::from_utf8_lossy(str_contents).to_string()), + span, + ty: Type::String, + }); + } + } + InterpolationMode::Expression => { + if token_start < end { + let span = Span { + start: token_start, + end, + }; + + let (expr, err) = parse_full_column_path(working_set, span); + error = error.or(err); + output.push(expr); + } + } + } + + if let Some(decl_id) = working_set.find_decl(b"build-string") { + ( + Expression { + expr: Expr::Call(Box::new(Call { + head: Span { + start: span.start, + end: span.start + 2, + }, + named: vec![], + positional: output, + decl_id, + })), + span, + ty: Type::String, }, error, ) + } else { + ( + Expression::garbage(span), + Some(ParseError::UnknownCommand(span)), + ) + } +} + +pub fn parse_variable_expr( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + let contents = working_set.get_span_contents(span); + + if contents == b"$true" { + return ( + Expression { + expr: Expr::Bool(true), + span, + ty: Type::Bool, + }, + None, + ); + } else if contents == b"$false" { + return ( + Expression { + expr: Expr::Bool(false), + span, + ty: Type::Bool, + }, + None, + ); } - pub fn parse_string(&mut self, span: Span) -> (Expression, Option) { - let bytes = self.get_span_contents(span); - let bytes = if (bytes.starts_with(b"\"") && bytes.ends_with(b"\"") && bytes.len() > 1) - || (bytes.starts_with(b"\'") && bytes.ends_with(b"\'") && bytes.len() > 1) - { - &bytes[1..(bytes.len() - 1)] - } else { - bytes - }; + let (id, err) = parse_variable(working_set, span); - if let Ok(token) = String::from_utf8(bytes.into()) { + if err.is_none() { + if let Some(id) = id { ( Expression { - expr: Expr::String(token), + expr: Expr::Var(id), span, - ty: Type::String, + ty: working_set.get_variable(id).clone(), }, None, ) } else { - ( - garbage(span), - Some(ParseError::Expected("string".into(), span)), - ) - } - } - - //TODO: Handle error case - pub fn parse_shape_name(&self, bytes: &[u8], span: Span) -> (SyntaxShape, Option) { - let result = match bytes { - b"any" => SyntaxShape::Any, - b"string" => SyntaxShape::String, - b"column-path" => SyntaxShape::ColumnPath, - b"number" => SyntaxShape::Number, - b"range" => SyntaxShape::Range, - b"int" => SyntaxShape::Int, - b"path" => SyntaxShape::FilePath, - b"glob" => SyntaxShape::GlobPattern, - b"block" => SyntaxShape::Block, - b"cond" => SyntaxShape::RowCondition, - b"operator" => SyntaxShape::Operator, - b"math" => SyntaxShape::MathExpression, - b"variable" => SyntaxShape::Variable, - b"signature" => SyntaxShape::Signature, - b"expr" => SyntaxShape::Expression, - _ => return (SyntaxShape::Any, Some(ParseError::UnknownType(span))), - }; - - (result, None) - } - - pub fn parse_type(&self, bytes: &[u8]) -> Type { - if bytes == b"int" { - Type::Int - } else { - Type::Unknown - } - } - - pub fn parse_var_with_opt_type( - &mut self, - spans: &[Span], - spans_idx: &mut usize, - ) -> (Expression, Option) { - let bytes = self.get_span_contents(spans[*spans_idx]).to_vec(); - - if bytes.ends_with(b":") { - // We end with colon, so the next span should be the type - if *spans_idx + 1 < spans.len() { - *spans_idx += 1; - let type_bytes = self.get_span_contents(spans[*spans_idx]); - - let ty = self.parse_type(type_bytes); - - let id = self.add_variable(bytes[0..(bytes.len() - 1)].to_vec(), ty.clone()); - - ( - Expression { - expr: Expr::Var(id), - span: span(&spans[*spans_idx - 1..*spans_idx + 1]), - ty, - }, - None, - ) - } else { - let id = self.add_variable(bytes[0..(bytes.len() - 1)].to_vec(), Type::Unknown); - ( - Expression { - expr: Expr::Var(id), - span: spans[*spans_idx], - ty: Type::Unknown, - }, - Some(ParseError::MissingType(spans[*spans_idx])), - ) - } - } else { - let id = self.add_variable(bytes, Type::Unknown); - + let name = working_set.get_span_contents(span).to_vec(); + // this seems okay to set it to unknown here, but we should double-check + let id = working_set.add_variable(name, Type::Unknown); ( Expression { expr: Expr::Var(id), - span: span(&spans[*spans_idx..*spans_idx + 1]), + span, ty: Type::Unknown, }, None, ) } + } else { + (garbage(span), err) } - pub fn parse_row_condition(&mut self, spans: &[Span]) -> (Expression, Option) { - self.parse_math_expression(spans) +} + +pub fn parse_full_column_path( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + // FIXME: assume for now a paren expr, but needs more + let bytes = working_set.get_span_contents(span); + let mut error = None; + + let mut start = span.start; + let mut end = span.end; + + if bytes.starts_with(b"(") { + start += 1; + } + if bytes.ends_with(b")") { + end -= 1; + } else { + error = error.or_else(|| { + Some(ParseError::Unclosed( + ")".into(), + Span { + start: end, + end: end + 1, + }, + )) + }); } - pub fn parse_signature(&mut self, span: Span) -> (Expression, Option) { - enum ParseMode { - ArgMode, - TypeMode, - } + let span = Span { start, end }; - enum Arg { - Positional(PositionalArg, bool), // bool - required - Flag(Flag), - } + let source = working_set.get_span_contents(span); - let bytes = self.get_span_contents(span); + let (output, err) = lex(source, start, &[], &[]); + error = error.or(err); - let mut error = None; - let mut start = span.start; - let mut end = span.end; + let (output, err) = lite_parse(&output); + error = error.or(err); - if bytes.starts_with(b"[") { - start += 1; - } - if bytes.ends_with(b"]") { - end -= 1; - } else { - error = error.or_else(|| { - Some(ParseError::Unclosed( - "]".into(), - Span { - start: end, - end: end + 1, - }, - )) - }); - } + let (output, err) = parse_block(working_set, &output, true); + error = error.or(err); - let span = Span { start, end }; - let source = self.get_span_contents(span); + let block_id = working_set.add_block(output); - let (output, err) = lex(source, span.start, &[b'\n', b','], &[b':']); - error = error.or(err); + ( + Expression { + expr: Expr::Subexpression(block_id), + span, + ty: Type::Unknown, // FIXME + }, + error, + ) +} - let mut args: Vec = vec![]; - let mut parse_mode = ParseMode::ArgMode; - - for token in &output { - match token { - Token { - contents: crate::TokenContents::Item, - span, - } => { - let span = *span; - let contents = self.get_span_contents(span); - - if contents == b":" { - match parse_mode { - ParseMode::ArgMode => { - parse_mode = ParseMode::TypeMode; - } - ParseMode::TypeMode => { - // We're seeing two types for the same thing for some reason, error - error = error - .or_else(|| Some(ParseError::Expected("type".into(), span))); - } - } - } else { - match parse_mode { - ParseMode::ArgMode => { - if contents.starts_with(b"--") && contents.len() > 2 { - // Long flag - let flags: Vec<_> = contents - .split(|x| x == &b'(') - .map(|x| x.to_vec()) - .collect(); - - let long = String::from_utf8_lossy(&flags[0]).to_string(); - let variable_name = flags[0][2..].to_vec(); - let var_id = self.add_variable(variable_name, Type::Unknown); - - if flags.len() == 1 { - args.push(Arg::Flag(Flag { - arg: None, - desc: String::new(), - long, - short: None, - required: false, - var_id: Some(var_id), - })); - } else { - let short_flag = &flags[1]; - let short_flag = if !short_flag.starts_with(b"-") - || !short_flag.ends_with(b")") - { - error = error.or_else(|| { - Some(ParseError::Expected( - "short flag".into(), - span, - )) - }); - short_flag - } else { - &short_flag[1..(short_flag.len() - 1)] - }; - - let short_flag = - String::from_utf8_lossy(short_flag).to_string(); - let chars: Vec = short_flag.chars().collect(); - let long = String::from_utf8_lossy(&flags[0]).to_string(); - let variable_name = flags[0][2..].to_vec(); - let var_id = - self.add_variable(variable_name, Type::Unknown); - - if chars.len() == 1 { - args.push(Arg::Flag(Flag { - arg: None, - desc: String::new(), - long, - short: Some(chars[0]), - required: false, - var_id: Some(var_id), - })); - } else { - error = error.or_else(|| { - Some(ParseError::Expected( - "short flag".into(), - span, - )) - }); - } - } - } else if contents.starts_with(b"-") && contents.len() > 1 { - // Short flag - - let short_flag = &contents[1..]; - let short_flag = - String::from_utf8_lossy(short_flag).to_string(); - let chars: Vec = short_flag.chars().collect(); - - if chars.len() > 1 { - error = error.or_else(|| { - Some(ParseError::Expected("short flag".into(), span)) - }); - - args.push(Arg::Flag(Flag { - arg: None, - desc: String::new(), - long: String::new(), - short: None, - required: false, - var_id: None, - })); - } else { - let mut encoded_var_name = vec![0u8; 4]; - let len = chars[0].encode_utf8(&mut encoded_var_name).len(); - let variable_name = encoded_var_name[0..len].to_vec(); - let var_id = - self.add_variable(variable_name, Type::Unknown); - - args.push(Arg::Flag(Flag { - arg: None, - desc: String::new(), - long: String::new(), - short: Some(chars[0]), - required: false, - var_id: Some(var_id), - })); - } - } else if contents.starts_with(b"(-") { - let short_flag = &contents[2..]; - - let short_flag = if !short_flag.ends_with(b")") { - error = error.or_else(|| { - Some(ParseError::Expected("short flag".into(), span)) - }); - short_flag - } else { - &short_flag[..(short_flag.len() - 1)] - }; - - let short_flag = - String::from_utf8_lossy(short_flag).to_string(); - let chars: Vec = short_flag.chars().collect(); - - if chars.len() == 1 { - match args.last_mut() { - Some(Arg::Flag(flag)) => { - if flag.short.is_some() { - error = error.or_else(|| { - Some(ParseError::Expected( - "one short flag".into(), - span, - )) - }); - } else { - flag.short = Some(chars[0]); - } - } - _ => { - error = error.or_else(|| { - Some(ParseError::Expected( - "unknown flag".into(), - span, - )) - }); - } - } - } else { - error = error.or_else(|| { - Some(ParseError::Expected("short flag".into(), span)) - }); - } - } else if contents.ends_with(b"?") { - let contents: Vec<_> = contents[..(contents.len() - 1)].into(); - let name = String::from_utf8_lossy(&contents).to_string(); - - let var_id = self.add_variable(contents, Type::Unknown); - - // Positional arg, optional - args.push(Arg::Positional( - PositionalArg { - desc: String::new(), - name, - shape: SyntaxShape::Any, - var_id: Some(var_id), - }, - false, - )) - } else { - let name = String::from_utf8_lossy(contents).to_string(); - let contents_vec = contents.to_vec(); - - let var_id = self.add_variable(contents_vec, Type::Unknown); - - // Positional arg, required - args.push(Arg::Positional( - PositionalArg { - desc: String::new(), - name, - shape: SyntaxShape::Any, - var_id: Some(var_id), - }, - true, - )) - } - } - ParseMode::TypeMode => { - if let Some(last) = args.last_mut() { - let (syntax_shape, err) = self.parse_shape_name(contents, span); - error = error.or(err); - //TODO check if we're replacing one already - match last { - Arg::Positional( - PositionalArg { shape, var_id, .. }, - .., - ) => { - self.set_variable_type(var_id.expect("internal error: all custom parameters must have var_ids"), syntax_shape.to_type()); - *shape = syntax_shape; - } - Arg::Flag(Flag { arg, var_id, .. }) => { - self.set_variable_type(var_id.expect("internal error: all custom parameters must have var_ids"), syntax_shape.to_type()); - *arg = Some(syntax_shape) - } - } - } - parse_mode = ParseMode::ArgMode; - } - } - } - } - Token { - contents: crate::TokenContents::Comment, - span, - } => { - let contents = self.get_span_contents(Span { - start: span.start + 1, - end: span.end, - }); - - let mut contents = String::from_utf8_lossy(contents).to_string(); - contents = contents.trim().into(); - - if let Some(last) = args.last_mut() { - match last { - Arg::Flag(flag) => { - if !flag.desc.is_empty() { - flag.desc.push('\n'); - } - flag.desc.push_str(&contents); - } - Arg::Positional(positional, ..) => { - if !positional.desc.is_empty() { - positional.desc.push('\n'); - } - positional.desc.push_str(&contents); - } - } - } - } - _ => {} - } - } - - let mut sig = Signature::new(String::new()); - - for arg in args { - match arg { - Arg::Positional(positional, required) => { - if positional.name.starts_with("...") { - let name = positional.name[3..].to_string(); - if name.is_empty() { - error = error.or(Some(ParseError::RestNeedsName(span))) - } else if sig.rest_positional.is_none() { - sig.rest_positional = Some(PositionalArg { name, ..positional }) - } else { - // Too many rest params - error = error.or(Some(ParseError::MultipleRestParams(span))) - } - } else if required { - sig.required_positional.push(positional) - } else { - sig.optional_positional.push(positional) - } - } - Arg::Flag(flag) => sig.named.push(flag), - } - } +pub fn parse_string( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(span); + let bytes = if (bytes.starts_with(b"\"") && bytes.ends_with(b"\"") && bytes.len() > 1) + || (bytes.starts_with(b"\'") && bytes.ends_with(b"\'") && bytes.len() > 1) + { + &bytes[1..(bytes.len() - 1)] + } else { + bytes + }; + if let Ok(token) = String::from_utf8(bytes.into()) { ( Expression { - expr: Expr::Signature(Box::new(sig)), + expr: Expr::String(token), span, - ty: Type::Unknown, + ty: Type::String, }, - error, + None, ) - } - - pub fn parse_list_expression( - &mut self, - span: Span, - element_shape: &SyntaxShape, - ) -> (Expression, Option) { - let bytes = self.get_span_contents(span); - - let mut error = None; - - let mut start = span.start; - let mut end = span.end; - - if bytes.starts_with(b"[") { - start += 1; - } - if bytes.ends_with(b"]") { - end -= 1; - } else { - error = error.or_else(|| { - Some(ParseError::Unclosed( - "]".into(), - Span { - start: end, - end: end + 1, - }, - )) - }); - } - - let span = Span { start, end }; - let source = self.get_span_contents(span); - - let (output, err) = lex(source, span.start, &[b'\n', b','], &[]); - error = error.or(err); - - let (output, err) = lite_parse(&output); - error = error.or(err); - - let mut args = vec![]; - - let mut contained_type: Option = None; - - if !output.block.is_empty() { - for arg in &output.block[0].commands { - let mut spans_idx = 0; - - while spans_idx < arg.parts.len() { - let (arg, err) = - self.parse_multispan_value(&arg.parts, &mut spans_idx, element_shape); - error = error.or(err); - - if let Some(ref ctype) = contained_type { - if *ctype != arg.ty { - contained_type = Some(Type::Unknown); - } - } else { - contained_type = Some(arg.ty.clone()); - } - - args.push(arg); - - spans_idx += 1; - } - } - } - + } else { ( - Expression { - expr: Expr::List(args), - span, - ty: Type::List(Box::new(if let Some(ty) = contained_type { - ty.clone() - } else { - Type::Unknown - })), - }, - error, + garbage(span), + Some(ParseError::Expected("string".into(), span)), ) } +} - pub fn parse_table_expression(&mut self, span: Span) -> (Expression, Option) { - let bytes = self.get_span_contents(span); - let mut error = None; +//TODO: Handle error case +pub fn parse_shape_name( + working_set: &StateWorkingSet, + bytes: &[u8], + span: Span, +) -> (SyntaxShape, Option) { + let result = match bytes { + b"any" => SyntaxShape::Any, + b"string" => SyntaxShape::String, + b"column-path" => SyntaxShape::ColumnPath, + b"number" => SyntaxShape::Number, + b"range" => SyntaxShape::Range, + b"int" => SyntaxShape::Int, + b"path" => SyntaxShape::FilePath, + b"glob" => SyntaxShape::GlobPattern, + b"block" => SyntaxShape::Block, + b"cond" => SyntaxShape::RowCondition, + b"operator" => SyntaxShape::Operator, + b"math" => SyntaxShape::MathExpression, + b"variable" => SyntaxShape::Variable, + b"signature" => SyntaxShape::Signature, + b"expr" => SyntaxShape::Expression, + _ => return (SyntaxShape::Any, Some(ParseError::UnknownType(span))), + }; - let mut start = span.start; - let mut end = span.end; + (result, None) +} - if bytes.starts_with(b"[") { - start += 1; - } - if bytes.ends_with(b"]") { - end -= 1; - } else { - error = error.or_else(|| { - Some(ParseError::Unclosed( - "]".into(), - Span { - start: end, - end: end + 1, - }, - )) - }); - } +pub fn parse_type(working_set: &StateWorkingSet, bytes: &[u8]) -> Type { + if bytes == b"int" { + Type::Int + } else { + Type::Unknown + } +} - let span = Span { start, end }; +pub fn parse_var_with_opt_type( + working_set: &mut StateWorkingSet, + spans: &[Span], + spans_idx: &mut usize, +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(spans[*spans_idx]).to_vec(); - let source = self.get_span_contents(span); + if bytes.ends_with(b":") { + // We end with colon, so the next span should be the type + if *spans_idx + 1 < spans.len() { + *spans_idx += 1; + let type_bytes = working_set.get_span_contents(spans[*spans_idx]); - let (output, err) = lex(source, start, &[b'\n', b','], &[]); - error = error.or(err); + let ty = parse_type(working_set, type_bytes); - let (output, err) = lite_parse(&output); - error = error.or(err); + let id = working_set.add_variable(bytes[0..(bytes.len() - 1)].to_vec(), ty.clone()); - match output.block.len() { - 0 => ( + ( Expression { - expr: Expr::List(vec![]), - span, - ty: Type::Table, + expr: Expr::Var(id), + span: span(&spans[*spans_idx - 1..*spans_idx + 1]), + ty, }, None, - ), - 1 => { - // List - self.parse_list_expression(span, &SyntaxShape::Any) - } - _ => { - let mut table_headers = vec![]; - - let (headers, err) = self.parse_value( - output.block[0].commands[0].parts[0], - &SyntaxShape::List(Box::new(SyntaxShape::Any)), - ); - error = error.or(err); - - if let Expression { - expr: Expr::List(headers), - .. - } = headers - { - table_headers = headers; - } - - let mut rows = vec![]; - for part in &output.block[1].commands[0].parts { - let (values, err) = - self.parse_value(*part, &SyntaxShape::List(Box::new(SyntaxShape::Any))); - error = error.or(err); - if let Expression { - expr: Expr::List(values), - .. - } = values - { - rows.push(values); - } - } - - ( - Expression { - expr: Expr::Table(table_headers, rows), - span, - ty: Type::Table, - }, - error, - ) - } - } - } - - pub fn parse_block_expression(&mut self, span: Span) -> (Expression, Option) { - let bytes = self.get_span_contents(span); - let mut error = None; - - let mut start = span.start; - let mut end = span.end; - - if bytes.starts_with(b"{") { - start += 1; + ) } else { - return ( - garbage(span), - Some(ParseError::Expected("block".into(), span)), - ); + let id = working_set.add_variable(bytes[0..(bytes.len() - 1)].to_vec(), Type::Unknown); + ( + Expression { + expr: Expr::Var(id), + span: spans[*spans_idx], + ty: Type::Unknown, + }, + Some(ParseError::MissingType(spans[*spans_idx])), + ) } - if bytes.ends_with(b"}") { - end -= 1; - } else { - error = error.or_else(|| { - Some(ParseError::Unclosed( - "}".into(), - Span { - start: end, - end: end + 1, - }, - )) - }); - } - - let span = Span { start, end }; - - let source = self.get_span_contents(span); - - let (output, err) = lex(source, start, &[], &[]); - error = error.or(err); - - // Check to see if we have parameters - let _params = if matches!( - output.first(), - Some(Token { - contents: TokenContents::Pipe, - .. - }) - ) { - // We've found a parameter list - let mut param_tokens = vec![]; - let mut token_iter = output.iter().skip(1); - for token in &mut token_iter { - if matches!( - token, - Token { - contents: TokenContents::Pipe, - .. - } - ) { - break; - } else { - param_tokens.push(token); - } - } - }; - - let (output, err) = lite_parse(&output); - error = error.or(err); - - let (output, err) = self.parse_block(&output, true); - error = error.or(err); - - let block_id = self.add_block(output); + } else { + let id = working_set.add_variable(bytes, Type::Unknown); ( Expression { - expr: Expr::Block(block_id), - span, - ty: Type::Block, - }, - error, - ) - } - - pub fn parse_value( - &mut self, - span: Span, - shape: &SyntaxShape, - ) -> (Expression, Option) { - let bytes = self.get_span_contents(span); - - // First, check the special-cases. These will likely represent specific values as expressions - // and may fit a variety of shapes. - // - // We check variable first because immediately following we check for variables with column paths - // which might result in a value that fits other shapes (and require the variable to already be - // declared) - if shape == &SyntaxShape::Variable { - return self.parse_variable_expr(span); - } else if bytes.starts_with(b"$") { - return self.parse_dollar_expr(span); - } else if bytes.starts_with(b"(") { - return self.parse_full_column_path(span); - } else if bytes.starts_with(b"[") { - match shape { - SyntaxShape::Any - | SyntaxShape::List(_) - | SyntaxShape::Table - | SyntaxShape::Signature => {} - _ => { - return ( - Expression::garbage(span), - Some(ParseError::Expected("non-[] value".into(), span)), - ); - } - } - } - - match shape { - SyntaxShape::Number => { - if let Ok(token) = String::from_utf8(bytes.into()) { - self.parse_number(&token, span) - } else { - ( - garbage(span), - Some(ParseError::Expected("number".into(), span)), - ) - } - } - SyntaxShape::Int => { - if let Ok(token) = String::from_utf8(bytes.into()) { - self.parse_int(&token, span) - } else { - ( - garbage(span), - Some(ParseError::Expected("int".into(), span)), - ) - } - } - SyntaxShape::String | SyntaxShape::GlobPattern | SyntaxShape::FilePath => { - self.parse_string(span) - } - SyntaxShape::Block => { - if bytes.starts_with(b"{") { - self.parse_block_expression(span) - } else { - ( - Expression::garbage(span), - Some(ParseError::Expected("block".into(), span)), - ) - } - } - SyntaxShape::Signature => { - if bytes.starts_with(b"[") { - self.parse_signature(span) - } else { - ( - Expression::garbage(span), - Some(ParseError::Expected("signature".into(), span)), - ) - } - } - SyntaxShape::List(elem) => { - if bytes.starts_with(b"[") { - self.parse_list_expression(span, elem) - } else { - ( - Expression::garbage(span), - Some(ParseError::Expected("list".into(), span)), - ) - } - } - SyntaxShape::Table => { - if bytes.starts_with(b"[") { - self.parse_table_expression(span) - } else { - ( - Expression::garbage(span), - Some(ParseError::Expected("table".into(), span)), - ) - } - } - SyntaxShape::Any => { - let shapes = [ - SyntaxShape::Int, - SyntaxShape::Number, - SyntaxShape::Range, - SyntaxShape::Filesize, - SyntaxShape::Duration, - SyntaxShape::Block, - SyntaxShape::Table, - SyntaxShape::List(Box::new(SyntaxShape::Any)), - SyntaxShape::String, - ]; - for shape in shapes.iter() { - if let (s, None) = self.parse_value(span, shape) { - return (s, None); - } - } - ( - garbage(span), - Some(ParseError::Expected("any shape".into(), span)), - ) - } - _ => (garbage(span), Some(ParseError::IncompleteParser(span))), - } - } - - pub fn parse_operator(&mut self, span: Span) -> (Expression, Option) { - let contents = self.get_span_contents(span); - - let operator = match contents { - b"==" => Operator::Equal, - b"!=" => Operator::NotEqual, - b"<" => Operator::LessThan, - b"<=" => Operator::LessThanOrEqual, - b">" => Operator::GreaterThan, - b">=" => Operator::GreaterThanOrEqual, - b"=~" => Operator::Contains, - b"!~" => Operator::NotContains, - b"+" => Operator::Plus, - b"-" => Operator::Minus, - b"*" => Operator::Multiply, - b"/" => Operator::Divide, - b"in" => Operator::In, - b"not-in" => Operator::NotIn, - b"mod" => Operator::Modulo, - b"&&" => Operator::And, - b"||" => Operator::Or, - b"**" => Operator::Pow, - _ => { - return ( - garbage(span), - Some(ParseError::Expected("operator".into(), span)), - ); - } - }; - - ( - Expression { - expr: Expr::Operator(operator), - span, + expr: Expr::Var(id), + span: span(&spans[*spans_idx..*spans_idx + 1]), ty: Type::Unknown, }, None, ) } +} +pub fn parse_row_condition( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Expression, Option) { + parse_math_expression(working_set, spans) +} - pub fn parse_math_expression(&mut self, spans: &[Span]) -> (Expression, Option) { - // As the expr_stack grows, we increase the required precedence to grow larger - // If, at any time, the operator we're looking at is the same or lower precedence - // of what is in the expression stack, we collapse the expression stack. - // - // This leads to an expression stack that grows under increasing precedence and collapses - // under decreasing/sustained precedence - // - // The end result is a stack that we can fold into binary operations as right associations - // safely. +pub fn parse_signature( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + enum ParseMode { + ArgMode, + TypeMode, + } - let mut expr_stack: Vec = vec![]; + enum Arg { + Positional(PositionalArg, bool), // bool - required + Flag(Flag), + } - let mut idx = 0; - let mut last_prec = 1000000; + let bytes = working_set.get_span_contents(span); - let mut error = None; - let (lhs, err) = self.parse_value(spans[0], &SyntaxShape::Any); - error = error.or(err); - idx += 1; + let mut error = None; + let mut start = span.start; + let mut end = span.end; - expr_stack.push(lhs); + if bytes.starts_with(b"[") { + start += 1; + } + if bytes.ends_with(b"]") { + end -= 1; + } else { + error = error.or_else(|| { + Some(ParseError::Unclosed( + "]".into(), + Span { + start: end, + end: end + 1, + }, + )) + }); + } - while idx < spans.len() { - let (op, err) = self.parse_operator(spans[idx]); - error = error.or(err); + let span = Span { start, end }; + let source = working_set.get_span_contents(span); - let op_prec = op.precedence(); + let (output, err) = lex(source, span.start, &[b'\n', b','], &[b':']); + error = error.or(err); - idx += 1; + let mut args: Vec = vec![]; + let mut parse_mode = ParseMode::ArgMode; - if idx == spans.len() { - // Handle broken math expr `1 +` etc - error = error.or(Some(ParseError::IncompleteMathExpression(spans[idx - 1]))); + for token in &output { + match token { + Token { + contents: crate::TokenContents::Item, + span, + } => { + let span = *span; + let contents = working_set.get_span_contents(span); - expr_stack.push(Expression::garbage(spans[idx - 1])); - expr_stack.push(Expression::garbage(spans[idx - 1])); + if contents == b":" { + match parse_mode { + ParseMode::ArgMode => { + parse_mode = ParseMode::TypeMode; + } + ParseMode::TypeMode => { + // We're seeing two types for the same thing for some reason, error + error = + error.or_else(|| Some(ParseError::Expected("type".into(), span))); + } + } + } else { + match parse_mode { + ParseMode::ArgMode => { + if contents.starts_with(b"--") && contents.len() > 2 { + // Long flag + let flags: Vec<_> = + contents.split(|x| x == &b'(').map(|x| x.to_vec()).collect(); - break; - } + let long = String::from_utf8_lossy(&flags[0]).to_string(); + let variable_name = flags[0][2..].to_vec(); + let var_id = working_set.add_variable(variable_name, Type::Unknown); - let (rhs, err) = self.parse_value(spans[idx], &SyntaxShape::Any); - error = error.or(err); + if flags.len() == 1 { + args.push(Arg::Flag(Flag { + arg: None, + desc: String::new(), + long, + short: None, + required: false, + var_id: Some(var_id), + })); + } else { + let short_flag = &flags[1]; + let short_flag = if !short_flag.starts_with(b"-") + || !short_flag.ends_with(b")") + { + error = error.or_else(|| { + Some(ParseError::Expected("short flag".into(), span)) + }); + short_flag + } else { + &short_flag[1..(short_flag.len() - 1)] + }; - if op_prec <= last_prec { - while expr_stack.len() > 1 { - // Collapse the right associated operations first - // so that we can get back to a stack with a lower precedence - let mut rhs = expr_stack - .pop() - .expect("internal error: expression stack empty"); - let mut op = expr_stack - .pop() - .expect("internal error: expression stack empty"); - let mut lhs = expr_stack - .pop() - .expect("internal error: expression stack empty"); + let short_flag = + String::from_utf8_lossy(short_flag).to_string(); + let chars: Vec = short_flag.chars().collect(); + let long = String::from_utf8_lossy(&flags[0]).to_string(); + let variable_name = flags[0][2..].to_vec(); + let var_id = + working_set.add_variable(variable_name, Type::Unknown); - let (result_ty, err) = self.math_result_type(&mut lhs, &mut op, &mut rhs); - error = error.or(err); + if chars.len() == 1 { + args.push(Arg::Flag(Flag { + arg: None, + desc: String::new(), + long, + short: Some(chars[0]), + required: false, + var_id: Some(var_id), + })); + } else { + error = error.or_else(|| { + Some(ParseError::Expected("short flag".into(), span)) + }); + } + } + } else if contents.starts_with(b"-") && contents.len() > 1 { + // Short flag - let op_span = span(&[lhs.span, rhs.span]); - expr_stack.push(Expression { - expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)), - span: op_span, - ty: result_ty, - }); + let short_flag = &contents[1..]; + let short_flag = String::from_utf8_lossy(short_flag).to_string(); + let chars: Vec = short_flag.chars().collect(); + + if chars.len() > 1 { + error = error.or_else(|| { + Some(ParseError::Expected("short flag".into(), span)) + }); + + args.push(Arg::Flag(Flag { + arg: None, + desc: String::new(), + long: String::new(), + short: None, + required: false, + var_id: None, + })); + } else { + let mut encoded_var_name = vec![0u8; 4]; + let len = chars[0].encode_utf8(&mut encoded_var_name).len(); + let variable_name = encoded_var_name[0..len].to_vec(); + let var_id = + working_set.add_variable(variable_name, Type::Unknown); + + args.push(Arg::Flag(Flag { + arg: None, + desc: String::new(), + long: String::new(), + short: Some(chars[0]), + required: false, + var_id: Some(var_id), + })); + } + } else if contents.starts_with(b"(-") { + let short_flag = &contents[2..]; + + let short_flag = if !short_flag.ends_with(b")") { + error = error.or_else(|| { + Some(ParseError::Expected("short flag".into(), span)) + }); + short_flag + } else { + &short_flag[..(short_flag.len() - 1)] + }; + + let short_flag = String::from_utf8_lossy(short_flag).to_string(); + let chars: Vec = short_flag.chars().collect(); + + if chars.len() == 1 { + match args.last_mut() { + Some(Arg::Flag(flag)) => { + if flag.short.is_some() { + error = error.or_else(|| { + Some(ParseError::Expected( + "one short flag".into(), + span, + )) + }); + } else { + flag.short = Some(chars[0]); + } + } + _ => { + error = error.or_else(|| { + Some(ParseError::Expected( + "unknown flag".into(), + span, + )) + }); + } + } + } else { + error = error.or_else(|| { + Some(ParseError::Expected("short flag".into(), span)) + }); + } + } else if contents.ends_with(b"?") { + let contents: Vec<_> = contents[..(contents.len() - 1)].into(); + let name = String::from_utf8_lossy(&contents).to_string(); + + let var_id = working_set.add_variable(contents, Type::Unknown); + + // Positional arg, optional + args.push(Arg::Positional( + PositionalArg { + desc: String::new(), + name, + shape: SyntaxShape::Any, + var_id: Some(var_id), + }, + false, + )) + } else { + let name = String::from_utf8_lossy(contents).to_string(); + let contents_vec = contents.to_vec(); + + let var_id = working_set.add_variable(contents_vec, Type::Unknown); + + // Positional arg, required + args.push(Arg::Positional( + PositionalArg { + desc: String::new(), + name, + shape: SyntaxShape::Any, + var_id: Some(var_id), + }, + true, + )) + } + } + ParseMode::TypeMode => { + if let Some(last) = args.last_mut() { + let (syntax_shape, err) = + parse_shape_name(working_set, contents, span); + error = error.or(err); + //TODO check if we're replacing one already + match last { + Arg::Positional(PositionalArg { shape, var_id, .. }, ..) => { + working_set.set_variable_type(var_id.expect("internal error: all custom parameters must have var_ids"), syntax_shape.to_type()); + *shape = syntax_shape; + } + Arg::Flag(Flag { arg, var_id, .. }) => { + working_set.set_variable_type(var_id.expect("internal error: all custom parameters must have var_ids"), syntax_shape.to_type()); + *arg = Some(syntax_shape) + } + } + } + parse_mode = ParseMode::ArgMode; + } + } } } - expr_stack.push(op); - expr_stack.push(rhs); + Token { + contents: crate::TokenContents::Comment, + span, + } => { + let contents = working_set.get_span_contents(Span { + start: span.start + 1, + end: span.end, + }); - last_prec = op_prec; + let mut contents = String::from_utf8_lossy(contents).to_string(); + contents = contents.trim().into(); - idx += 1; + if let Some(last) = args.last_mut() { + match last { + Arg::Flag(flag) => { + if !flag.desc.is_empty() { + flag.desc.push('\n'); + } + flag.desc.push_str(&contents); + } + Arg::Positional(positional, ..) => { + if !positional.desc.is_empty() { + positional.desc.push('\n'); + } + positional.desc.push_str(&contents); + } + } + } + } + _ => {} } + } - while expr_stack.len() != 1 { - let mut rhs = expr_stack - .pop() - .expect("internal error: expression stack empty"); - let mut op = expr_stack - .pop() - .expect("internal error: expression stack empty"); - let mut lhs = expr_stack - .pop() - .expect("internal error: expression stack empty"); + let mut sig = Signature::new(String::new()); - let (result_ty, err) = self.math_result_type(&mut lhs, &mut op, &mut rhs); + for arg in args { + match arg { + Arg::Positional(positional, required) => { + if positional.name.starts_with("...") { + let name = positional.name[3..].to_string(); + if name.is_empty() { + error = error.or(Some(ParseError::RestNeedsName(span))) + } else if sig.rest_positional.is_none() { + sig.rest_positional = Some(PositionalArg { name, ..positional }) + } else { + // Too many rest params + error = error.or(Some(ParseError::MultipleRestParams(span))) + } + } else if required { + sig.required_positional.push(positional) + } else { + sig.optional_positional.push(positional) + } + } + Arg::Flag(flag) => sig.named.push(flag), + } + } + + ( + Expression { + expr: Expr::Signature(Box::new(sig)), + span, + ty: Type::Unknown, + }, + error, + ) +} + +pub fn parse_list_expression( + working_set: &mut StateWorkingSet, + span: Span, + element_shape: &SyntaxShape, +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(span); + + let mut error = None; + + let mut start = span.start; + let mut end = span.end; + + if bytes.starts_with(b"[") { + start += 1; + } + if bytes.ends_with(b"]") { + end -= 1; + } else { + error = error.or_else(|| { + Some(ParseError::Unclosed( + "]".into(), + Span { + start: end, + end: end + 1, + }, + )) + }); + } + + let span = Span { start, end }; + let source = working_set.get_span_contents(span); + + let (output, err) = lex(source, span.start, &[b'\n', b','], &[]); + error = error.or(err); + + let (output, err) = lite_parse(&output); + error = error.or(err); + + let mut args = vec![]; + + let mut contained_type: Option = None; + + if !output.block.is_empty() { + for arg in &output.block[0].commands { + let mut spans_idx = 0; + + while spans_idx < arg.parts.len() { + let (arg, err) = + parse_multispan_value(working_set, &arg.parts, &mut spans_idx, element_shape); + error = error.or(err); + + if let Some(ref ctype) = contained_type { + if *ctype != arg.ty { + contained_type = Some(Type::Unknown); + } + } else { + contained_type = Some(arg.ty.clone()); + } + + args.push(arg); + + spans_idx += 1; + } + } + } + + ( + Expression { + expr: Expr::List(args), + span, + ty: Type::List(Box::new(if let Some(ty) = contained_type { + ty.clone() + } else { + Type::Unknown + })), + }, + error, + ) +} + +pub fn parse_table_expression( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(span); + let mut error = None; + + let mut start = span.start; + let mut end = span.end; + + if bytes.starts_with(b"[") { + start += 1; + } + if bytes.ends_with(b"]") { + end -= 1; + } else { + error = error.or_else(|| { + Some(ParseError::Unclosed( + "]".into(), + Span { + start: end, + end: end + 1, + }, + )) + }); + } + + let span = Span { start, end }; + + let source = working_set.get_span_contents(span); + + let (output, err) = lex(source, start, &[b'\n', b','], &[]); + error = error.or(err); + + let (output, err) = lite_parse(&output); + error = error.or(err); + + match output.block.len() { + 0 => ( + Expression { + expr: Expr::List(vec![]), + span, + ty: Type::Table, + }, + None, + ), + 1 => { + // List + parse_list_expression(working_set, span, &SyntaxShape::Any) + } + _ => { + let mut table_headers = vec![]; + + let (headers, err) = parse_value( + working_set, + output.block[0].commands[0].parts[0], + &SyntaxShape::List(Box::new(SyntaxShape::Any)), + ); error = error.or(err); - let binary_op_span = span(&[lhs.span, rhs.span]); - expr_stack.push(Expression { - expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)), - span: binary_op_span, - ty: result_ty, - }); + if let Expression { + expr: Expr::List(headers), + .. + } = headers + { + table_headers = headers; + } + + let mut rows = vec![]; + for part in &output.block[1].commands[0].parts { + let (values, err) = parse_value( + working_set, + *part, + &SyntaxShape::List(Box::new(SyntaxShape::Any)), + ); + error = error.or(err); + if let Expression { + expr: Expr::List(values), + .. + } = values + { + rows.push(values); + } + } + + ( + Expression { + expr: Expr::Table(table_headers, rows), + span, + ty: Type::Table, + }, + error, + ) + } + } +} + +pub fn parse_block_expression( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(span); + let mut error = None; + + let mut start = span.start; + let mut end = span.end; + + if bytes.starts_with(b"{") { + start += 1; + } else { + return ( + garbage(span), + Some(ParseError::Expected("block".into(), span)), + ); + } + if bytes.ends_with(b"}") { + end -= 1; + } else { + error = error.or_else(|| { + Some(ParseError::Unclosed( + "}".into(), + Span { + start: end, + end: end + 1, + }, + )) + }); + } + + let span = Span { start, end }; + + let source = working_set.get_span_contents(span); + + let (output, err) = lex(source, start, &[], &[]); + error = error.or(err); + + // Check to see if we have parameters + let _params = if matches!( + output.first(), + Some(Token { + contents: TokenContents::Pipe, + .. + }) + ) { + // We've found a parameter list + let mut param_tokens = vec![]; + let mut token_iter = output.iter().skip(1); + for token in &mut token_iter { + if matches!( + token, + Token { + contents: TokenContents::Pipe, + .. + } + ) { + break; + } else { + param_tokens.push(token); + } + } + }; + + let (output, err) = lite_parse(&output); + error = error.or(err); + + let (output, err) = parse_block(working_set, &output, true); + error = error.or(err); + + let block_id = working_set.add_block(output); + + ( + Expression { + expr: Expr::Block(block_id), + span, + ty: Type::Block, + }, + error, + ) +} + +pub fn parse_value( + working_set: &mut StateWorkingSet, + span: Span, + shape: &SyntaxShape, +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(span); + + // First, check the special-cases. These will likely represent specific values as expressions + // and may fit a variety of shapes. + // + // We check variable first because immediately following we check for variables with column paths + // which might result in a value that fits other shapes (and require the variable to already be + // declared) + if shape == &SyntaxShape::Variable { + return parse_variable_expr(working_set, span); + } else if bytes.starts_with(b"$") { + return parse_dollar_expr(working_set, span); + } else if bytes.starts_with(b"(") { + return parse_full_column_path(working_set, span); + } else if bytes.starts_with(b"[") { + match shape { + SyntaxShape::Any + | SyntaxShape::List(_) + | SyntaxShape::Table + | SyntaxShape::Signature => {} + _ => { + return ( + Expression::garbage(span), + Some(ParseError::Expected("non-[] value".into(), span)), + ); + } + } + } + + match shape { + SyntaxShape::Number => { + if let Ok(token) = String::from_utf8(bytes.into()) { + parse_number(working_set, &token, span) + } else { + ( + garbage(span), + Some(ParseError::Expected("number".into(), span)), + ) + } + } + SyntaxShape::Int => { + if let Ok(token) = String::from_utf8(bytes.into()) { + parse_int(working_set, &token, span) + } else { + ( + garbage(span), + Some(ParseError::Expected("int".into(), span)), + ) + } + } + SyntaxShape::String | SyntaxShape::GlobPattern | SyntaxShape::FilePath => { + parse_string(working_set, span) + } + SyntaxShape::Block => { + if bytes.starts_with(b"{") { + parse_block_expression(working_set, span) + } else { + ( + Expression::garbage(span), + Some(ParseError::Expected("block".into(), span)), + ) + } + } + SyntaxShape::Signature => { + if bytes.starts_with(b"[") { + parse_signature(working_set, span) + } else { + ( + Expression::garbage(span), + Some(ParseError::Expected("signature".into(), span)), + ) + } + } + SyntaxShape::List(elem) => { + if bytes.starts_with(b"[") { + parse_list_expression(working_set, span, elem) + } else { + ( + Expression::garbage(span), + Some(ParseError::Expected("list".into(), span)), + ) + } + } + SyntaxShape::Table => { + if bytes.starts_with(b"[") { + parse_table_expression(working_set, span) + } else { + ( + Expression::garbage(span), + Some(ParseError::Expected("table".into(), span)), + ) + } + } + SyntaxShape::Any => { + let shapes = [ + SyntaxShape::Int, + SyntaxShape::Number, + SyntaxShape::Range, + SyntaxShape::Filesize, + SyntaxShape::Duration, + SyntaxShape::Block, + SyntaxShape::Table, + SyntaxShape::List(Box::new(SyntaxShape::Any)), + SyntaxShape::String, + ]; + for shape in shapes.iter() { + if let (s, None) = parse_value(working_set, span, shape) { + return (s, None); + } + } + ( + garbage(span), + Some(ParseError::Expected("any shape".into(), span)), + ) + } + _ => (garbage(span), Some(ParseError::IncompleteParser(span))), + } +} + +pub fn parse_operator( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Expression, Option) { + let contents = working_set.get_span_contents(span); + + let operator = match contents { + b"==" => Operator::Equal, + b"!=" => Operator::NotEqual, + b"<" => Operator::LessThan, + b"<=" => Operator::LessThanOrEqual, + b">" => Operator::GreaterThan, + b">=" => Operator::GreaterThanOrEqual, + b"=~" => Operator::Contains, + b"!~" => Operator::NotContains, + b"+" => Operator::Plus, + b"-" => Operator::Minus, + b"*" => Operator::Multiply, + b"/" => Operator::Divide, + b"in" => Operator::In, + b"not-in" => Operator::NotIn, + b"mod" => Operator::Modulo, + b"&&" => Operator::And, + b"||" => Operator::Or, + b"**" => Operator::Pow, + _ => { + return ( + garbage(span), + Some(ParseError::Expected("operator".into(), span)), + ); + } + }; + + ( + Expression { + expr: Expr::Operator(operator), + span, + ty: Type::Unknown, + }, + None, + ) +} + +pub fn parse_math_expression( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Expression, Option) { + // As the expr_stack grows, we increase the required precedence to grow larger + // If, at any time, the operator we're looking at is the same or lower precedence + // of what is in the expression stack, we collapse the expression stack. + // + // This leads to an expression stack that grows under increasing precedence and collapses + // under decreasing/sustained precedence + // + // The end result is a stack that we can fold into binary operations as right associations + // safely. + + let mut expr_stack: Vec = vec![]; + + let mut idx = 0; + let mut last_prec = 1000000; + + let mut error = None; + let (lhs, err) = parse_value(working_set, spans[0], &SyntaxShape::Any); + error = error.or(err); + idx += 1; + + expr_stack.push(lhs); + + while idx < spans.len() { + let (op, err) = parse_operator(working_set, spans[idx]); + error = error.or(err); + + let op_prec = op.precedence(); + + idx += 1; + + if idx == spans.len() { + // Handle broken math expr `1 +` etc + error = error.or(Some(ParseError::IncompleteMathExpression(spans[idx - 1]))); + + expr_stack.push(Expression::garbage(spans[idx - 1])); + expr_stack.push(Expression::garbage(spans[idx - 1])); + + break; } - let output = expr_stack + let (rhs, err) = parse_value(working_set, spans[idx], &SyntaxShape::Any); + error = error.or(err); + + if op_prec <= last_prec { + while expr_stack.len() > 1 { + // Collapse the right associated operations first + // so that we can get back to a stack with a lower precedence + let mut rhs = expr_stack + .pop() + .expect("internal error: expression stack empty"); + let mut op = expr_stack + .pop() + .expect("internal error: expression stack empty"); + let mut lhs = expr_stack + .pop() + .expect("internal error: expression stack empty"); + + let (result_ty, err) = math_result_type(working_set, &mut lhs, &mut op, &mut rhs); + error = error.or(err); + + let op_span = span(&[lhs.span, rhs.span]); + expr_stack.push(Expression { + expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)), + span: op_span, + ty: result_ty, + }); + } + } + expr_stack.push(op); + expr_stack.push(rhs); + + last_prec = op_prec; + + idx += 1; + } + + while expr_stack.len() != 1 { + let mut rhs = expr_stack + .pop() + .expect("internal error: expression stack empty"); + let mut op = expr_stack + .pop() + .expect("internal error: expression stack empty"); + let mut lhs = expr_stack .pop() .expect("internal error: expression stack empty"); - (output, error) + let (result_ty, err) = math_result_type(working_set, &mut lhs, &mut op, &mut rhs); + error = error.or(err); + + let binary_op_span = span(&[lhs.span, rhs.span]); + expr_stack.push(Expression { + expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)), + span: binary_op_span, + ty: result_ty, + }); } - pub fn parse_expression(&mut self, spans: &[Span]) -> (Expression, Option) { - let bytes = self.get_span_contents(spans[0]); + let output = expr_stack + .pop() + .expect("internal error: expression stack empty"); - match bytes[0] { - b'0' | b'1' | b'2' | b'3' | b'4' | b'5' | b'6' | b'7' | b'8' | b'9' | b'(' | b'{' - | b'[' | b'$' | b'"' | b'\'' => self.parse_math_expression(spans), - _ => self.parse_call(spans, true), - } + (output, error) +} + +pub fn parse_expression( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Expression, Option) { + let bytes = working_set.get_span_contents(spans[0]); + + match bytes[0] { + b'0' | b'1' | b'2' | b'3' | b'4' | b'5' | b'6' | b'7' | b'8' | b'9' | b'(' | b'{' + | b'[' | b'$' | b'"' | b'\'' => parse_math_expression(working_set, spans), + _ => parse_call(working_set, spans, true), } +} - pub fn parse_variable(&mut self, span: Span) -> (Option, Option) { - let bytes = self.get_span_contents(span); +pub fn parse_variable( + working_set: &mut StateWorkingSet, + span: Span, +) -> (Option, Option) { + let bytes = working_set.get_span_contents(span); - if is_variable(bytes) { - if let Some(var_id) = self.find_variable(bytes) { - (Some(var_id), None) - } else { - (None, None) - } + if is_variable(bytes) { + if let Some(var_id) = working_set.find_variable(bytes) { + (Some(var_id), None) } else { - (None, Some(ParseError::Expected("variable".into(), span))) + (None, None) } + } else { + (None, Some(ParseError::Expected("variable".into(), span))) } +} - pub fn parse_def_predecl(&mut self, spans: &[Span]) { - let name = self.get_span_contents(spans[0]); +pub fn parse_def_predecl(working_set: &mut StateWorkingSet, spans: &[Span]) { + let name = working_set.get_span_contents(spans[0]); - if name == b"def" && spans.len() >= 4 { - let (name_expr, ..) = self.parse_string(spans[1]); - let name = name_expr.as_string(); + if name == b"def" && spans.len() >= 4 { + let (name_expr, ..) = parse_string(working_set, spans[1]); + let name = name_expr.as_string(); - self.enter_scope(); - // FIXME: because parse_signature will update the scope with the variables it sees - // we end up parsing the signature twice per def. The first time is during the predecl - // so that we can see the types that are part of the signature, which we need for parsing. - // The second time is when we actually parse the body itself. - // We can't reuse the first time because the variables that are created during parse_signature - // are lost when we exit the scope below. - let (sig, ..) = self.parse_signature(spans[2]); - let signature = sig.as_signature(); - self.exit_scope(); + working_set.enter_scope(); + // FIXME: because parse_signature will update the scope with the variables it sees + // we end up parsing the signature twice per def. The first time is during the predecl + // so that we can see the types that are part of the signature, which we need for parsing. + // The second time is when we actually parse the body itworking_set. + // We can't reuse the first time because the variables that are created during parse_signature + // are lost when we exit the scope below. + let (sig, ..) = parse_signature(working_set, spans[2]); + let signature = sig.as_signature(); + working_set.exit_scope(); - match (name, signature) { - (Some(name), Some(mut signature)) => { - signature.name = name; - let decl = Declaration { - signature, - body: None, - }; + match (name, signature) { + (Some(name), Some(mut signature)) => { + signature.name = name; + let decl = signature.predeclare(); - self.add_decl(decl); - } - _ => {} + working_set.add_decl(decl); } + _ => {} } } +} - pub fn parse_def(&mut self, spans: &[Span]) -> (Statement, Option) { - let mut error = None; - let name = self.get_span_contents(spans[0]); +pub fn parse_def( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + let mut error = None; + let name = working_set.get_span_contents(spans[0]); - if name == b"def" && spans.len() >= 4 { - //FIXME: don't use expect here - let (name_expr, err) = self.parse_string(spans[1]); - error = error.or(err); + if name == b"def" && spans.len() >= 4 { + //FIXME: don't use expect here + let (name_expr, err) = parse_string(working_set, spans[1]); + error = error.or(err); - self.enter_scope(); - let (sig, err) = self.parse_signature(spans[2]); - error = error.or(err); + working_set.enter_scope(); + let (sig, err) = parse_signature(working_set, spans[2]); + error = error.or(err); - let (block, err) = self.parse_block_expression(spans[3]); - error = error.or(err); - self.exit_scope(); + let (block, err) = parse_block_expression(working_set, spans[3]); + error = error.or(err); + working_set.exit_scope(); - let name = name_expr.as_string(); + let name = name_expr.as_string(); - let signature = sig.as_signature(); + let signature = sig.as_signature(); - let block_id = block.as_block(); + let block_id = block.as_block(); - match (name, signature, block_id) { - (Some(name), Some(mut signature), Some(block_id)) => { - let decl_id = self - .find_decl(name.as_bytes()) - .expect("internal error: predeclaration failed to add definition"); + match (name, signature, block_id) { + (Some(name), Some(mut signature), Some(block_id)) => { + let decl_id = working_set + .find_decl(name.as_bytes()) + .expect("internal error: predeclaration failed to add definition"); - let declaration = self.get_decl_mut(decl_id); + let declaration = working_set.get_decl_mut(decl_id); - signature.name = name; - declaration.signature = signature; - declaration.body = Some(block_id); + signature.name = name; - let def_decl_id = self - .find_decl(b"def") - .expect("internal error: missing def command"); + *declaration = signature.into_block_command(block_id); - let call = Box::new(Call { - head: spans[0], - decl_id: def_decl_id, - positional: vec![name_expr, sig, block], - named: vec![], - }); + let def_decl_id = working_set + .find_decl(b"def") + .expect("internal error: missing def command"); - ( - Statement::Expression(Expression { - expr: Expr::Call(call), - span: span(spans), - ty: Type::Unknown, - }), - error, - ) - } - _ => ( + let call = Box::new(Call { + head: spans[0], + decl_id: def_decl_id, + positional: vec![name_expr, sig, block], + named: vec![], + }); + + ( Statement::Expression(Expression { - expr: Expr::Garbage, + expr: Expr::Call(call), span: span(spans), ty: Type::Unknown, }), error, - ), + ) } - } else { - ( + _ => ( Statement::Expression(Expression { expr: Expr::Garbage, span: span(spans), ty: Type::Unknown, }), - Some(ParseError::UnknownState( - "internal error: definition unparseable".into(), - span(spans), - )), - ) + error, + ), } - } - - pub fn parse_alias(&mut self, spans: &[Span]) -> (Statement, Option) { - let name = self.get_span_contents(spans[0]); - - if name == b"alias" { - if let Some(decl_id) = self.find_decl(b"alias") { - let (call, call_span, _) = self.parse_internal_call(spans[0], &spans[1..], decl_id); - - if spans.len() >= 4 { - let alias_name = self.get_span_contents(spans[1]); - - let alias_name = if alias_name.starts_with(b"\"") - && alias_name.ends_with(b"\"") - && alias_name.len() > 1 - { - alias_name[1..(alias_name.len() - 1)].to_vec() - } else { - alias_name.to_vec() - }; - let _equals = self.get_span_contents(spans[2]); - - let replacement = spans[3..].to_vec(); - - //println!("{:?} {:?}", alias_name, replacement); - - self.add_alias(alias_name, replacement); - } - - return ( - Statement::Expression(Expression { - expr: Expr::Call(call), - span: call_span, - ty: Type::Unknown, - }), - None, - ); - } - } - + } else { ( Statement::Expression(Expression { expr: Expr::Garbage, @@ -2334,154 +2093,219 @@ impl<'a> ParserWorkingSet<'a> { ty: Type::Unknown, }), Some(ParseError::UnknownState( - "internal error: let statement unparseable".into(), + "internal error: definition unparseable".into(), span(spans), )), ) } - - pub fn parse_let(&mut self, spans: &[Span]) -> (Statement, Option) { - let name = self.get_span_contents(spans[0]); - - if name == b"let" { - if let Some(decl_id) = self.find_decl(b"let") { - let (call, call_span, err) = - self.parse_internal_call(spans[0], &spans[1..], decl_id); - - // Update the variable to the known type if we can. - if err.is_none() { - let var_id = call.positional[0] - .as_var() - .expect("internal error: expected variable"); - let rhs_type = call.positional[1].ty.clone(); - - self.set_variable_type(var_id, rhs_type); - } - - return ( - Statement::Expression(Expression { - expr: Expr::Call(call), - span: call_span, - ty: Type::Unknown, - }), - err, - ); - } - } - ( - Statement::Expression(Expression { - expr: Expr::Garbage, - span: span(spans), - ty: Type::Unknown, - }), - Some(ParseError::UnknownState( - "internal error: let statement unparseable".into(), - span(spans), - )), - ) - } - - pub fn parse_statement(&mut self, spans: &[Span]) -> (Statement, Option) { - // FIXME: improve errors by checking keyword first - if let (decl, None) = self.parse_def(spans) { - (decl, None) - } else if let (stmt, None) = self.parse_let(spans) { - (stmt, None) - } else if let (stmt, None) = self.parse_alias(spans) { - (stmt, None) - } else { - let (expr, err) = self.parse_expression(spans); - (Statement::Expression(expr), err) - } - } - - pub fn parse_block( - &mut self, - lite_block: &LiteBlock, - scoped: bool, - ) -> (Block, Option) { - let mut error = None; - if scoped { - self.enter_scope(); - } - - let mut block = Block::new(); - - // Pre-declare any definition so that definitions - // that share the same block can see each other - for pipeline in &lite_block.block { - if pipeline.commands.len() == 1 { - self.parse_def_predecl(&pipeline.commands[0].parts); - } - } - - for pipeline in &lite_block.block { - if pipeline.commands.len() > 1 { - let mut output = vec![]; - for command in &pipeline.commands { - let (expr, err) = self.parse_expression(&command.parts); - error = error.or(err); - - output.push(expr); - } - block.stmts.push(Statement::Pipeline(Pipeline { - expressions: output, - })); - } else { - let (stmt, err) = self.parse_statement(&pipeline.commands[0].parts); - error = error.or(err); - - block.stmts.push(stmt); - } - } - - if scoped { - self.exit_scope(); - } - - (block, error) - } - - pub fn parse_file( - &mut self, - fname: &str, - contents: &[u8], - scoped: bool, - ) -> (Block, Option) { - let mut error = None; - - let span_offset = self.next_span_start(); - - self.add_file(fname.into(), contents); - - let (output, err) = lex(contents, span_offset, &[], &[]); - error = error.or(err); - - let (output, err) = lite_parse(&output); - error = error.or(err); - - let (output, err) = self.parse_block(&output, scoped); - error = error.or(err); - - (output, error) - } - - pub fn parse_source(&mut self, source: &[u8], scoped: bool) -> (Block, Option) { - let mut error = None; - - let span_offset = self.next_span_start(); - - self.add_file("source".into(), source); - - let (output, err) = lex(source, span_offset, &[], &[]); - error = error.or(err); - - let (output, err) = lite_parse(&output); - error = error.or(err); - - let (output, err) = self.parse_block(&output, scoped); - error = error.or(err); - - (output, error) - } +} + +pub fn parse_alias( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + let name = working_set.get_span_contents(spans[0]); + + if name == b"alias" { + if let Some(decl_id) = working_set.find_decl(b"alias") { + let (call, call_span, _) = + parse_internal_call(working_set, spans[0], &spans[1..], decl_id); + + if spans.len() >= 4 { + let alias_name = working_set.get_span_contents(spans[1]); + + let alias_name = if alias_name.starts_with(b"\"") + && alias_name.ends_with(b"\"") + && alias_name.len() > 1 + { + alias_name[1..(alias_name.len() - 1)].to_vec() + } else { + alias_name.to_vec() + }; + let _equals = working_set.get_span_contents(spans[2]); + + let replacement = spans[3..].to_vec(); + + //println!("{:?} {:?}", alias_name, replacement); + + working_set.add_alias(alias_name, replacement); + } + + return ( + Statement::Expression(Expression { + expr: Expr::Call(call), + span: call_span, + ty: Type::Unknown, + }), + None, + ); + } + } + + ( + Statement::Expression(Expression { + expr: Expr::Garbage, + span: span(spans), + ty: Type::Unknown, + }), + Some(ParseError::UnknownState( + "internal error: let statement unparseable".into(), + span(spans), + )), + ) +} + +pub fn parse_let( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + let name = working_set.get_span_contents(spans[0]); + + if name == b"let" { + if let Some(decl_id) = working_set.find_decl(b"let") { + let (call, call_span, err) = + parse_internal_call(working_set, spans[0], &spans[1..], decl_id); + + // Update the variable to the known type if we can. + if err.is_none() { + let var_id = call.positional[0] + .as_var() + .expect("internal error: expected variable"); + let rhs_type = call.positional[1].ty.clone(); + + working_set.set_variable_type(var_id, rhs_type); + } + + return ( + Statement::Expression(Expression { + expr: Expr::Call(call), + span: call_span, + ty: Type::Unknown, + }), + err, + ); + } + } + ( + Statement::Expression(Expression { + expr: Expr::Garbage, + span: span(spans), + ty: Type::Unknown, + }), + Some(ParseError::UnknownState( + "internal error: let statement unparseable".into(), + span(spans), + )), + ) +} + +pub fn parse_statement( + working_set: &mut StateWorkingSet, + spans: &[Span], +) -> (Statement, Option) { + // FIXME: improve errors by checking keyword first + if let (decl, None) = parse_def(working_set, spans) { + (decl, None) + } else if let (stmt, None) = parse_let(working_set, spans) { + (stmt, None) + } else if let (stmt, None) = parse_alias(working_set, spans) { + (stmt, None) + } else { + let (expr, err) = parse_expression(working_set, spans); + (Statement::Expression(expr), err) + } +} + +pub fn parse_block( + working_set: &mut StateWorkingSet, + lite_block: &LiteBlock, + scoped: bool, +) -> (Block, Option) { + let mut error = None; + if scoped { + working_set.enter_scope(); + } + + let mut block = Block::new(); + + // Pre-declare any definition so that definitions + // that share the same block can see each other + for pipeline in &lite_block.block { + if pipeline.commands.len() == 1 { + parse_def_predecl(working_set, &pipeline.commands[0].parts); + } + } + + for pipeline in &lite_block.block { + if pipeline.commands.len() > 1 { + let mut output = vec![]; + for command in &pipeline.commands { + let (expr, err) = parse_expression(working_set, &command.parts); + error = error.or(err); + + output.push(expr); + } + block.stmts.push(Statement::Pipeline(Pipeline { + expressions: output, + })); + } else { + let (stmt, err) = parse_statement(working_set, &pipeline.commands[0].parts); + error = error.or(err); + + block.stmts.push(stmt); + } + } + + if scoped { + working_set.exit_scope(); + } + + (block, error) +} + +pub fn parse_file( + working_set: &mut StateWorkingSet, + fname: &str, + contents: &[u8], + scoped: bool, +) -> (Block, Option) { + let mut error = None; + + let span_offset = working_set.next_span_start(); + + working_set.add_file(fname.into(), contents); + + let (output, err) = lex(contents, span_offset, &[], &[]); + error = error.or(err); + + let (output, err) = lite_parse(&output); + error = error.or(err); + + let (output, err) = parse_block(working_set, &output, scoped); + error = error.or(err); + + (output, error) +} + +pub fn parse_source( + working_set: &mut StateWorkingSet, + source: &[u8], + scoped: bool, +) -> (Block, Option) { + let mut error = None; + + let span_offset = working_set.next_span_start(); + + working_set.add_file("source".into(), source); + + let (output, err) = lex(source, span_offset, &[], &[]); + error = error.or(err); + + let (output, err) = lite_parse(&output); + error = error.or(err); + + let (output, err) = parse_block(working_set, &output, scoped); + error = error.or(err); + + (output, error) } diff --git a/crates/nu-parser/src/type_check.rs b/crates/nu-parser/src/type_check.rs index 65b0697e96..0e88dac135 100644 --- a/crates/nu-parser/src/type_check.rs +++ b/crates/nu-parser/src/type_check.rs @@ -1,251 +1,46 @@ -use crate::{parser::Operator, Expr, Expression, ParseError, ParserWorkingSet}; -use nu_protocol::Type; +use crate::ParseError; +use nu_protocol::{Expr, Expression, Operator, StateWorkingSet, Type}; -impl<'a> ParserWorkingSet<'a> { - pub fn type_compatible(lhs: &Type, rhs: &Type) -> bool { - match (lhs, rhs) { - (Type::List(c), Type::List(d)) => ParserWorkingSet::type_compatible(c, d), - (Type::Unknown, _) => true, - (_, Type::Unknown) => true, - (lhs, rhs) => lhs == rhs, - } +pub fn type_compatible(lhs: &Type, rhs: &Type) -> bool { + match (lhs, rhs) { + (Type::List(c), Type::List(d)) => type_compatible(c, d), + (Type::Unknown, _) => true, + (_, Type::Unknown) => true, + (lhs, rhs) => lhs == rhs, } +} - pub fn math_result_type( - &self, - lhs: &mut Expression, - op: &mut Expression, - rhs: &mut Expression, - ) -> (Type, Option) { - match &op.expr { - Expr::Operator(operator) => match operator { - Operator::Plus => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Int, None), - (Type::Float, Type::Int) => (Type::Float, None), - (Type::Int, Type::Float) => (Type::Float, None), - (Type::Float, Type::Float) => (Type::Float, None), - (Type::String, Type::String) => (Type::String, None), - (Type::Unknown, _) => (Type::Unknown, None), - (_, Type::Unknown) => (Type::Unknown, None), - (Type::Int, _) => { - *rhs = Expression::garbage(rhs.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::Minus => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Int, None), - (Type::Float, Type::Int) => (Type::Float, None), - (Type::Int, Type::Float) => (Type::Float, None), - (Type::Float, Type::Float) => (Type::Float, None), - (Type::Unknown, _) => (Type::Unknown, None), - (_, Type::Unknown) => (Type::Unknown, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::Multiply => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Int, None), - (Type::Float, Type::Int) => (Type::Float, None), - (Type::Int, Type::Float) => (Type::Float, None), - (Type::Float, Type::Float) => (Type::Float, None), - (Type::Unknown, _) => (Type::Unknown, None), - (_, Type::Unknown) => (Type::Unknown, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::Divide => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Int, None), - (Type::Float, Type::Int) => (Type::Float, None), - (Type::Int, Type::Float) => (Type::Float, None), - (Type::Float, Type::Float) => (Type::Float, None), - (Type::Unknown, _) => (Type::Unknown, None), - (_, Type::Unknown) => (Type::Unknown, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::LessThan => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Bool, None), - (Type::Float, Type::Int) => (Type::Bool, None), - (Type::Int, Type::Float) => (Type::Bool, None), - (Type::Float, Type::Float) => (Type::Bool, None), - (Type::Unknown, _) => (Type::Bool, None), - (_, Type::Unknown) => (Type::Bool, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::LessThanOrEqual => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Bool, None), - (Type::Float, Type::Int) => (Type::Bool, None), - (Type::Int, Type::Float) => (Type::Bool, None), - (Type::Float, Type::Float) => (Type::Bool, None), - (Type::Unknown, _) => (Type::Bool, None), - (_, Type::Unknown) => (Type::Bool, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::GreaterThan => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Bool, None), - (Type::Float, Type::Int) => (Type::Bool, None), - (Type::Int, Type::Float) => (Type::Bool, None), - (Type::Float, Type::Float) => (Type::Bool, None), - (Type::Unknown, _) => (Type::Bool, None), - (_, Type::Unknown) => (Type::Bool, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::GreaterThanOrEqual => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Bool, None), - (Type::Float, Type::Int) => (Type::Bool, None), - (Type::Int, Type::Float) => (Type::Bool, None), - (Type::Float, Type::Float) => (Type::Bool, None), - (Type::Unknown, _) => (Type::Bool, None), - (_, Type::Unknown) => (Type::Bool, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::Equal => match (&lhs.ty, &rhs.ty) { - (Type::Float, Type::Int) => (Type::Bool, None), - (Type::Int, Type::Float) => (Type::Bool, None), - (x, y) if x == y => (Type::Bool, None), - (Type::Unknown, _) => (Type::Bool, None), - (_, Type::Unknown) => (Type::Bool, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - Operator::NotEqual => match (&lhs.ty, &rhs.ty) { - (Type::Int, Type::Int) => (Type::Bool, None), - (Type::Float, Type::Int) => (Type::Bool, None), - (Type::Int, Type::Float) => (Type::Bool, None), - (Type::Float, Type::Float) => (Type::Bool, None), - (Type::Unknown, _) => (Type::Bool, None), - (_, Type::Unknown) => (Type::Bool, None), - _ => { - *op = Expression::garbage(op.span); - ( - Type::Unknown, - Some(ParseError::UnsupportedOperation( - op.span, - lhs.span, - lhs.ty.clone(), - rhs.span, - rhs.ty.clone(), - )), - ) - } - }, - +pub fn math_result_type( + working_set: &StateWorkingSet, + lhs: &mut Expression, + op: &mut Expression, + rhs: &mut Expression, +) -> (Type, Option) { + match &op.expr { + Expr::Operator(operator) => match operator { + Operator::Plus => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Int, None), + (Type::Float, Type::Int) => (Type::Float, None), + (Type::Int, Type::Float) => (Type::Float, None), + (Type::Float, Type::Float) => (Type::Float, None), + (Type::String, Type::String) => (Type::String, None), + (Type::Unknown, _) => (Type::Unknown, None), + (_, Type::Unknown) => (Type::Unknown, None), + (Type::Int, _) => { + *rhs = Expression::garbage(rhs.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } _ => { *op = Expression::garbage(op.span); - ( Type::Unknown, Some(ParseError::UnsupportedOperation( @@ -258,14 +53,217 @@ impl<'a> ParserWorkingSet<'a> { ) } }, + Operator::Minus => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Int, None), + (Type::Float, Type::Int) => (Type::Float, None), + (Type::Int, Type::Float) => (Type::Float, None), + (Type::Float, Type::Float) => (Type::Float, None), + (Type::Unknown, _) => (Type::Unknown, None), + (_, Type::Unknown) => (Type::Unknown, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::Multiply => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Int, None), + (Type::Float, Type::Int) => (Type::Float, None), + (Type::Int, Type::Float) => (Type::Float, None), + (Type::Float, Type::Float) => (Type::Float, None), + (Type::Unknown, _) => (Type::Unknown, None), + (_, Type::Unknown) => (Type::Unknown, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::Divide => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Int, None), + (Type::Float, Type::Int) => (Type::Float, None), + (Type::Int, Type::Float) => (Type::Float, None), + (Type::Float, Type::Float) => (Type::Float, None), + (Type::Unknown, _) => (Type::Unknown, None), + (_, Type::Unknown) => (Type::Unknown, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::LessThan => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Bool, None), + (Type::Float, Type::Int) => (Type::Bool, None), + (Type::Int, Type::Float) => (Type::Bool, None), + (Type::Float, Type::Float) => (Type::Bool, None), + (Type::Unknown, _) => (Type::Bool, None), + (_, Type::Unknown) => (Type::Bool, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::LessThanOrEqual => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Bool, None), + (Type::Float, Type::Int) => (Type::Bool, None), + (Type::Int, Type::Float) => (Type::Bool, None), + (Type::Float, Type::Float) => (Type::Bool, None), + (Type::Unknown, _) => (Type::Bool, None), + (_, Type::Unknown) => (Type::Bool, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::GreaterThan => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Bool, None), + (Type::Float, Type::Int) => (Type::Bool, None), + (Type::Int, Type::Float) => (Type::Bool, None), + (Type::Float, Type::Float) => (Type::Bool, None), + (Type::Unknown, _) => (Type::Bool, None), + (_, Type::Unknown) => (Type::Bool, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::GreaterThanOrEqual => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Bool, None), + (Type::Float, Type::Int) => (Type::Bool, None), + (Type::Int, Type::Float) => (Type::Bool, None), + (Type::Float, Type::Float) => (Type::Bool, None), + (Type::Unknown, _) => (Type::Bool, None), + (_, Type::Unknown) => (Type::Bool, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::Equal => match (&lhs.ty, &rhs.ty) { + (Type::Float, Type::Int) => (Type::Bool, None), + (Type::Int, Type::Float) => (Type::Bool, None), + (x, y) if x == y => (Type::Bool, None), + (Type::Unknown, _) => (Type::Bool, None), + (_, Type::Unknown) => (Type::Bool, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + Operator::NotEqual => match (&lhs.ty, &rhs.ty) { + (Type::Int, Type::Int) => (Type::Bool, None), + (Type::Float, Type::Int) => (Type::Bool, None), + (Type::Int, Type::Float) => (Type::Bool, None), + (Type::Float, Type::Float) => (Type::Bool, None), + (Type::Unknown, _) => (Type::Bool, None), + (_, Type::Unknown) => (Type::Bool, None), + _ => { + *op = Expression::garbage(op.span); + ( + Type::Unknown, + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), + ) + } + }, + _ => { *op = Expression::garbage(op.span); ( Type::Unknown, - Some(ParseError::IncompleteMathExpression(op.span)), + Some(ParseError::UnsupportedOperation( + op.span, + lhs.span, + lhs.ty.clone(), + rhs.span, + rhs.ty.clone(), + )), ) } + }, + _ => { + *op = Expression::garbage(op.span); + + ( + Type::Unknown, + Some(ParseError::IncompleteMathExpression(op.span)), + ) } } } diff --git a/crates/nu-parser/tests/test_parser.rs b/crates/nu-parser/tests/test_parser.rs index ed38f3ccfd..e13075196b 100644 --- a/crates/nu-parser/tests/test_parser.rs +++ b/crates/nu-parser/tests/test_parser.rs @@ -1,11 +1,11 @@ +use nu_parser::ParseError; use nu_parser::*; -use nu_parser::{ParseError, ParserState}; -use nu_protocol::{Signature, SyntaxShape}; +use nu_protocol::{EngineState, Signature, SyntaxShape}; #[test] pub fn parse_int() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let (block, err) = working_set.parse_source(b"3", true); @@ -22,8 +22,8 @@ pub fn parse_int() { #[test] pub fn parse_call() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); working_set.add_decl(sig.into()); @@ -46,8 +46,8 @@ pub fn parse_call() { #[test] pub fn parse_call_missing_flag_arg() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); working_set.add_decl(sig.into()); @@ -58,8 +58,8 @@ pub fn parse_call_missing_flag_arg() { #[test] pub fn parse_call_missing_short_flag_arg() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); working_set.add_decl(sig.into()); @@ -70,8 +70,8 @@ pub fn parse_call_missing_short_flag_arg() { #[test] pub fn parse_call_too_many_shortflag_args() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo") .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')) @@ -86,8 +86,8 @@ pub fn parse_call_too_many_shortflag_args() { #[test] pub fn parse_call_unknown_shorthand() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").switch("--jazz", "jazz!!", Some('j')); working_set.add_decl(sig.into()); @@ -97,8 +97,8 @@ pub fn parse_call_unknown_shorthand() { #[test] pub fn parse_call_extra_positional() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").switch("--jazz", "jazz!!", Some('j')); working_set.add_decl(sig.into()); @@ -108,8 +108,8 @@ pub fn parse_call_extra_positional() { #[test] pub fn parse_call_missing_req_positional() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").required("jazz", SyntaxShape::Int, "jazz!!"); working_set.add_decl(sig.into()); @@ -119,8 +119,8 @@ pub fn parse_call_missing_req_positional() { #[test] pub fn parse_call_missing_req_flag() { - let parser_state = ParserState::new(); - let mut working_set = ParserWorkingSet::new(&parser_state); + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").required_named("--jazz", SyntaxShape::Int, "jazz!!", None); working_set.add_decl(sig.into()); diff --git a/crates/nu-protocol/Cargo.toml b/crates/nu-protocol/Cargo.toml index c1a398d04e..d6c800e68e 100644 --- a/crates/nu-protocol/Cargo.toml +++ b/crates/nu-protocol/Cargo.toml @@ -6,3 +6,4 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +codespan-reporting = "0.11.1" \ No newline at end of file diff --git a/crates/nu-protocol/src/block.rs b/crates/nu-protocol/src/block.rs new file mode 100644 index 0000000000..3b2f66bef1 --- /dev/null +++ b/crates/nu-protocol/src/block.rs @@ -0,0 +1,44 @@ +use std::ops::{Index, IndexMut}; + +use crate::Statement; + +#[derive(Debug, Clone)] +pub struct Block { + pub stmts: Vec, +} + +impl Block { + pub fn len(&self) -> usize { + self.stmts.len() + } + + pub fn is_empty(&self) -> bool { + self.stmts.is_empty() + } +} + +impl Index for Block { + type Output = Statement; + + fn index(&self, index: usize) -> &Self::Output { + &self.stmts[index] + } +} + +impl IndexMut for Block { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.stmts[index] + } +} + +impl Default for Block { + fn default() -> Self { + Self::new() + } +} + +impl Block { + pub fn new() -> Self { + Self { stmts: vec![] } + } +} diff --git a/crates/nu-protocol/src/call.rs b/crates/nu-protocol/src/call.rs new file mode 100644 index 0000000000..df90a12060 --- /dev/null +++ b/crates/nu-protocol/src/call.rs @@ -0,0 +1,27 @@ +use crate::{DeclId, Expression, Span}; + +#[derive(Debug, Clone)] +pub struct Call { + /// identifier of the declaration to call + pub decl_id: DeclId, + pub head: Span, + pub positional: Vec, + pub named: Vec<(String, Option)>, +} + +impl Default for Call { + fn default() -> Self { + Self::new() + } +} + +impl Call { + pub fn new() -> Call { + Self { + decl_id: 0, + head: Span::unknown(), + positional: vec![], + named: vec![], + } + } +} diff --git a/crates/nu-protocol/src/command.rs b/crates/nu-protocol/src/command.rs new file mode 100644 index 0000000000..08caf637c4 --- /dev/null +++ b/crates/nu-protocol/src/command.rs @@ -0,0 +1,60 @@ +use crate::{Example, Signature}; + +pub trait Command { + fn name(&self) -> &str; + + fn signature(&self) -> Signature { + Signature::new(self.name()).desc(self.usage()).filter() + } + + fn usage(&self) -> &str; + + fn extra_usage(&self) -> &str { + "" + } + + // fn run(&self, args: CommandArgs) -> Result { + // let context = args.context.clone(); + // let stream = self.run_with_actions(args)?; + + // Ok(Box::new(crate::evaluate::internal::InternalIterator { + // context, + // input: stream, + // leftovers: InputStream::empty(), + // }) + // .into_output_stream()) + // } + + fn is_binary(&self) -> bool { + false + } + + // Commands that are not meant to be run by users + fn is_private(&self) -> bool { + false + } + + fn examples(&self) -> Vec { + Vec::new() + } + + // This is a built-in command + fn is_builtin(&self) -> bool { + true + } + + // Is a sub command + fn is_sub(&self) -> bool { + self.name().contains(' ') + } + + // Is a plugin command + fn is_plugin(&self) -> bool { + false + } + + // Is a custom command i.e. def blah [] { } + fn is_custom(&self) -> bool { + false + } +} diff --git a/crates/nu-protocol/src/declaration.rs b/crates/nu-protocol/src/declaration.rs deleted file mode 100644 index 0bb638a33b..0000000000 --- a/crates/nu-protocol/src/declaration.rs +++ /dev/null @@ -1,6 +0,0 @@ -use crate::{BlockId, Signature}; - -pub struct Declaration { - pub signature: Box, - pub body: Option, -} diff --git a/crates/nu-parser/src/parser_state.rs b/crates/nu-protocol/src/engine_state.rs similarity index 72% rename from crates/nu-parser/src/parser_state.rs rename to crates/nu-protocol/src/engine_state.rs index b4d1c82d56..8d96c9bdab 100644 --- a/crates/nu-parser/src/parser_state.rs +++ b/crates/nu-protocol/src/engine_state.rs @@ -1,13 +1,12 @@ -use crate::parser::Block; +use crate::{Block, BlockId, Command, DeclId, Span, Type, VarId}; use core::panic; -use nu_protocol::{BlockId, DeclId, Declaration, Span, Type, VarId}; -use std::{collections::HashMap, slice::Iter}; +use std::{collections::HashMap, ops::Range, slice::Iter}; -pub struct ParserState { +pub struct EngineState { files: Vec<(String, usize, usize)>, file_contents: Vec, vars: Vec, - decls: Vec, + decls: Vec>, blocks: Vec, scope: Vec, } @@ -29,13 +28,13 @@ impl ScopeFrame { } } -impl Default for ParserState { +impl Default for EngineState { fn default() -> Self { Self::new() } } -impl ParserState { +impl EngineState { pub fn new() -> Self { Self { files: vec![], @@ -47,7 +46,7 @@ impl ParserState { } } - pub fn merge_delta(this: &mut ParserState, mut delta: ParserDelta) { + pub fn merge_delta(this: &mut EngineState, mut delta: StateDelta) { // Take the mutable reference and extend the permanent state from the working set this.files.extend(delta.files); this.file_contents.extend(delta.file_contents); @@ -93,7 +92,7 @@ impl ParserState { pub fn print_decls(&self) { for decl in self.decls.iter().enumerate() { - println!("decl{}: {:?}", decl.0, decl.1.signature); + println!("decl{}: {:?}", decl.0, decl.1.signature()); } } @@ -119,7 +118,7 @@ impl ParserState { .expect("internal error: missing variable") } - pub fn get_decl(&self, decl_id: DeclId) -> &Declaration { + pub fn get_decl(&self, decl_id: DeclId) -> &Box { self.decls .get(decl_id) .expect("internal error: missing declaration") @@ -176,21 +175,21 @@ impl ParserState { } } -pub struct ParserWorkingSet<'a> { - permanent_state: &'a ParserState, - pub delta: ParserDelta, +pub struct StateWorkingSet<'a> { + pub permanent_state: &'a EngineState, + pub delta: StateDelta, } -pub struct ParserDelta { +pub struct StateDelta { files: Vec<(String, usize, usize)>, pub(crate) file_contents: Vec, - vars: Vec, // indexed by VarId - decls: Vec, // indexed by DeclId - blocks: Vec, // indexed by BlockId + vars: Vec, // indexed by VarId + decls: Vec>, // indexed by DeclId + blocks: Vec, // indexed by BlockId scope: Vec, } -impl ParserDelta { +impl StateDelta { pub fn num_files(&self) -> usize { self.files.len() } @@ -212,10 +211,10 @@ impl ParserDelta { } } -impl<'a> ParserWorkingSet<'a> { - pub fn new(permanent_state: &'a ParserState) -> Self { +impl<'a> StateWorkingSet<'a> { + pub fn new(permanent_state: &'a EngineState) -> Self { Self { - delta: ParserDelta { + delta: StateDelta { files: vec![], file_contents: vec![], vars: vec![], @@ -239,8 +238,8 @@ impl<'a> ParserWorkingSet<'a> { self.delta.num_blocks() + self.permanent_state.num_blocks() } - pub fn add_decl(&mut self, decl: Declaration) -> DeclId { - let name = decl.signature.name.as_bytes().to_vec(); + pub fn add_decl(&mut self, decl: Box) -> DeclId { + let name = decl.name().as_bytes().to_vec(); self.delta.decls.push(decl); let decl_id = self.num_decls() - 1; @@ -346,10 +345,10 @@ impl<'a> ParserWorkingSet<'a> { None } - pub fn update_decl(&mut self, decl_id: usize, block: Option) { - let decl = self.get_decl_mut(decl_id); - decl.body = block; - } + // pub fn update_decl(&mut self, decl_id: usize, block: Option) { + // let decl = self.get_decl_mut(decl_id); + // decl.body = block; + // } pub fn contains_decl_partial_match(&self, name: &[u8]) -> bool { for scope in self.delta.scope.iter().rev() { @@ -460,7 +459,7 @@ impl<'a> ParserWorkingSet<'a> { } } - pub fn get_decl(&self, decl_id: DeclId) -> &Declaration { + pub fn get_decl(&self, decl_id: DeclId) -> &Box { let num_permanent_decls = self.permanent_state.num_decls(); if decl_id < num_permanent_decls { self.permanent_state.get_decl(decl_id) @@ -472,7 +471,7 @@ impl<'a> ParserWorkingSet<'a> { } } - pub fn get_decl_mut(&mut self, decl_id: DeclId) -> &mut Declaration { + pub fn get_decl_mut(&mut self, decl_id: DeclId) -> &mut Box { let num_permanent_decls = self.permanent_state.num_decls(); if decl_id < num_permanent_decls { panic!("internal error: can only mutate declarations in working set") @@ -496,30 +495,122 @@ impl<'a> ParserWorkingSet<'a> { } } - pub fn render(self) -> ParserDelta { + pub fn render(self) -> StateDelta { self.delta } } +impl<'a> codespan_reporting::files::Files<'a> for StateWorkingSet<'a> { + type FileId = usize; + + type Name = String; + + type Source = String; + + fn name(&'a self, id: Self::FileId) -> Result { + Ok(self.get_filename(id)) + } + + fn source( + &'a self, + id: Self::FileId, + ) -> Result { + Ok(self.get_file_source(id)) + } + + fn line_index( + &'a self, + id: Self::FileId, + byte_index: usize, + ) -> Result { + let source = self.get_file_source(id); + + let mut count = 0; + + for byte in source.bytes().enumerate() { + if byte.0 == byte_index { + // println!("count: {} for file: {} index: {}", count, id, byte_index); + return Ok(count); + } + if byte.1 == b'\n' { + count += 1; + } + } + + // println!("count: {} for file: {} index: {}", count, id, byte_index); + Ok(count) + } + + fn line_range( + &'a self, + id: Self::FileId, + line_index: usize, + ) -> Result, codespan_reporting::files::Error> { + let source = self.get_file_source(id); + + let mut count = 0; + + let mut start = Some(0); + let mut end = None; + + for byte in source.bytes().enumerate() { + #[allow(clippy::comparison_chain)] + if count > line_index { + let start = start.expect("internal error: couldn't find line"); + let end = end.expect("internal error: couldn't find line"); + + // println!( + // "Span: {}..{} for fileid: {} index: {}", + // start, end, id, line_index + // ); + return Ok(start..end); + } else if count == line_index { + end = Some(byte.0 + 1); + } + + #[allow(clippy::comparison_chain)] + if byte.1 == b'\n' { + count += 1; + if count > line_index { + break; + } else if count == line_index { + start = Some(byte.0 + 1); + } + } + } + + match (start, end) { + (Some(start), Some(end)) => { + // println!( + // "Span: {}..{} for fileid: {} index: {}", + // start, end, id, line_index + // ); + Ok(start..end) + } + _ => Err(codespan_reporting::files::Error::FileMissing), + } + } +} + #[cfg(test)] -mod parser_state_tests { +mod engine_state_tests { use super::*; #[test] fn add_file_gives_id() { - let parser_state = ParserState::new(); - let mut parser_state = ParserWorkingSet::new(&parser_state); - let id = parser_state.add_file("test.nu".into(), &[]); + let engine_state = EngineState::new(); + let mut engine_state = StateWorkingSet::new(&engine_state); + let id = engine_state.add_file("test.nu".into(), &[]); assert_eq!(id, 0); } #[test] fn add_file_gives_id_including_parent() { - let mut parser_state = ParserState::new(); - let parent_id = parser_state.add_file("test.nu".into(), vec![]); + let mut engine_state = EngineState::new(); + let parent_id = engine_state.add_file("test.nu".into(), vec![]); - let mut working_set = ParserWorkingSet::new(&parser_state); + let mut working_set = StateWorkingSet::new(&engine_state); let working_set_id = working_set.add_file("child.nu".into(), &[]); assert_eq!(parent_id, 0); @@ -528,19 +619,19 @@ mod parser_state_tests { #[test] fn merge_states() { - let mut parser_state = ParserState::new(); - parser_state.add_file("test.nu".into(), vec![]); + let mut engine_state = EngineState::new(); + engine_state.add_file("test.nu".into(), vec![]); let delta = { - let mut working_set = ParserWorkingSet::new(&parser_state); + let mut working_set = StateWorkingSet::new(&engine_state); working_set.add_file("child.nu".into(), &[]); working_set.render() }; - ParserState::merge_delta(&mut parser_state, delta); + EngineState::merge_delta(&mut engine_state, delta); - assert_eq!(parser_state.num_files(), 2); - assert_eq!(&parser_state.files[0].0, "test.nu"); - assert_eq!(&parser_state.files[1].0, "child.nu"); + assert_eq!(engine_state.num_files(), 2); + assert_eq!(&engine_state.files[0].0, "test.nu"); + assert_eq!(&engine_state.files[1].0, "child.nu"); } } diff --git a/crates/nu-protocol/src/example.rs b/crates/nu-protocol/src/example.rs new file mode 100644 index 0000000000..894b4b2876 --- /dev/null +++ b/crates/nu-protocol/src/example.rs @@ -0,0 +1,7 @@ +use crate::Value; + +pub struct Example { + pub example: &'static str, + pub description: &'static str, + pub result: Option>, +} diff --git a/crates/nu-protocol/src/expr.rs b/crates/nu-protocol/src/expr.rs new file mode 100644 index 0000000000..cfc74799c2 --- /dev/null +++ b/crates/nu-protocol/src/expr.rs @@ -0,0 +1,21 @@ +use crate::{BlockId, Call, Expression, Operator, Signature, Span, VarId}; + +#[derive(Debug, Clone)] +pub enum Expr { + Bool(bool), + Int(i64), + Float(f64), + Var(VarId), + Call(Box), + ExternalCall(Vec, Vec>), + Operator(Operator), + BinaryOp(Box, Box, Box), //lhs, op, rhs + Subexpression(BlockId), + Block(BlockId), + List(Vec), + Table(Vec, Vec>), + Keyword(Vec, Span, Box), + String(String), // FIXME: improve this in the future? + Signature(Box), + Garbage, +} diff --git a/crates/nu-protocol/src/expression.rs b/crates/nu-protocol/src/expression.rs new file mode 100644 index 0000000000..176bb65ce4 --- /dev/null +++ b/crates/nu-protocol/src/expression.rs @@ -0,0 +1,85 @@ +use crate::{BlockId, Expr, Operator, Signature, Span, Type, VarId}; + +#[derive(Debug, Clone)] +pub struct Expression { + pub expr: Expr, + pub span: Span, + pub ty: Type, +} +impl Expression { + pub fn garbage(span: Span) -> Expression { + Expression { + expr: Expr::Garbage, + span, + ty: Type::Unknown, + } + } + pub fn precedence(&self) -> usize { + match &self.expr { + Expr::Operator(operator) => { + // Higher precedence binds tighter + + match operator { + Operator::Pow => 100, + Operator::Multiply | Operator::Divide | Operator::Modulo => 95, + Operator::Plus | Operator::Minus => 90, + Operator::NotContains + | Operator::Contains + | Operator::LessThan + | Operator::LessThanOrEqual + | Operator::GreaterThan + | Operator::GreaterThanOrEqual + | Operator::Equal + | Operator::NotEqual + | Operator::In + | Operator::NotIn => 80, + Operator::And => 50, + Operator::Or => 40, // TODO: should we have And and Or be different precedence? + } + } + _ => 0, + } + } + + pub fn as_block(&self) -> Option { + match self.expr { + Expr::Block(block_id) => Some(block_id), + _ => None, + } + } + + pub fn as_signature(&self) -> Option> { + match &self.expr { + Expr::Signature(sig) => Some(sig.clone()), + _ => None, + } + } + + pub fn as_list(&self) -> Option> { + match &self.expr { + Expr::List(list) => Some(list.clone()), + _ => None, + } + } + + pub fn as_keyword(&self) -> Option<&Expression> { + match &self.expr { + Expr::Keyword(_, _, expr) => Some(expr), + _ => None, + } + } + + pub fn as_var(&self) -> Option { + match self.expr { + Expr::Var(var_id) => Some(var_id), + _ => None, + } + } + + pub fn as_string(&self) -> Option { + match &self.expr { + Expr::String(string) => Some(string.clone()), + _ => None, + } + } +} diff --git a/crates/nu-protocol/src/lib.rs b/crates/nu-protocol/src/lib.rs index 919fc2adf0..d4e1166cf4 100644 --- a/crates/nu-protocol/src/lib.rs +++ b/crates/nu-protocol/src/lib.rs @@ -1,17 +1,35 @@ -mod declaration; +mod block; +mod call; +mod command; +mod engine_state; +mod example; +mod expr; +mod expression; mod id; +mod operator; +mod pipeline; mod shell_error; mod signature; mod span; +mod statement; mod syntax_shape; mod ty; mod value; -pub use declaration::*; +pub use block::*; +pub use call::*; +pub use command::*; +pub use engine_state::*; +pub use example::*; +pub use expr::*; +pub use expression::*; pub use id::*; +pub use operator::*; +pub use pipeline::*; pub use shell_error::*; pub use signature::*; pub use span::*; +pub use statement::*; pub use syntax_shape::*; pub use ty::*; pub use value::*; diff --git a/crates/nu-protocol/src/operator.rs b/crates/nu-protocol/src/operator.rs new file mode 100644 index 0000000000..f230e4a89a --- /dev/null +++ b/crates/nu-protocol/src/operator.rs @@ -0,0 +1,48 @@ +use std::fmt::Display; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Operator { + Equal, + NotEqual, + LessThan, + GreaterThan, + LessThanOrEqual, + GreaterThanOrEqual, + Contains, + NotContains, + Plus, + Minus, + Multiply, + Divide, + In, + NotIn, + Modulo, + And, + Or, + Pow, +} + +impl Display for Operator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Operator::Equal => write!(f, "=="), + Operator::NotEqual => write!(f, "!="), + Operator::LessThan => write!(f, "<"), + Operator::GreaterThan => write!(f, ">"), + Operator::Contains => write!(f, "=~"), + Operator::NotContains => write!(f, "!~"), + Operator::Plus => write!(f, "+"), + Operator::Minus => write!(f, "-"), + Operator::Multiply => write!(f, "*"), + Operator::Divide => write!(f, "/"), + Operator::In => write!(f, "in"), + Operator::NotIn => write!(f, "not-in"), + Operator::Modulo => write!(f, "mod"), + Operator::And => write!(f, "&&"), + Operator::Or => write!(f, "||"), + Operator::Pow => write!(f, "**"), + Operator::LessThanOrEqual => write!(f, "<="), + Operator::GreaterThanOrEqual => write!(f, ">="), + } + } +} diff --git a/crates/nu-protocol/src/pipeline.rs b/crates/nu-protocol/src/pipeline.rs new file mode 100644 index 0000000000..cdeb722158 --- /dev/null +++ b/crates/nu-protocol/src/pipeline.rs @@ -0,0 +1,20 @@ +use crate::Expression; + +#[derive(Debug, Clone)] +pub struct Pipeline { + pub expressions: Vec, +} + +impl Default for Pipeline { + fn default() -> Self { + Self::new() + } +} + +impl Pipeline { + pub fn new() -> Self { + Self { + expressions: vec![], + } + } +} diff --git a/crates/nu-protocol/src/signature.rs b/crates/nu-protocol/src/signature.rs index f7e4ddea71..f5fc981223 100644 --- a/crates/nu-protocol/src/signature.rs +++ b/crates/nu-protocol/src/signature.rs @@ -1,5 +1,7 @@ +use crate::BlockId; +use crate::Command; +use crate::SyntaxShape; use crate::VarId; -use crate::{Declaration, SyntaxShape}; #[derive(Debug, Clone)] pub struct Flag { @@ -273,22 +275,62 @@ impl Signature { } None } -} -impl From> for Declaration { - fn from(val: Box) -> Self { - Declaration { - signature: val, - body: None, - } + /// Set the filter flag for the signature + pub fn filter(mut self) -> Signature { + self.is_filter = true; + self + } + + /// Create a placeholder implementation of Command as a way to predeclare a definition's + /// signature so other definitions can see it. This placeholder is later replaced with the + /// full definition in a second pass of the parser. + pub fn predeclare(self) -> Box { + Box::new(Predeclaration { signature: self }) + } + + /// Combines a signature and a block into a runnable block + pub fn into_block_command(self, block_id: BlockId) -> Box { + Box::new(BlockCommand { + signature: self, + block_id, + }) } } -impl From for Declaration { - fn from(val: Signature) -> Self { - Declaration { - signature: Box::new(val), - body: None, - } +struct Predeclaration { + signature: Signature, +} + +impl Command for Predeclaration { + fn name(&self) -> &str { + &self.signature.name + } + + fn signature(&self) -> Signature { + self.signature.clone() + } + + fn usage(&self) -> &str { + &self.signature.usage + } +} + +struct BlockCommand { + signature: Signature, + block_id: BlockId, +} + +impl Command for BlockCommand { + fn name(&self) -> &str { + &self.signature.name + } + + fn signature(&self) -> Signature { + self.signature.clone() + } + + fn usage(&self) -> &str { + &self.signature.usage } } diff --git a/crates/nu-protocol/src/statement.rs b/crates/nu-protocol/src/statement.rs new file mode 100644 index 0000000000..27b86f005e --- /dev/null +++ b/crates/nu-protocol/src/statement.rs @@ -0,0 +1,8 @@ +use crate::{DeclId, Expression, Pipeline}; + +#[derive(Debug, Clone)] +pub enum Statement { + Declaration(DeclId), + Pipeline(Pipeline), + Expression(Expression), +} diff --git a/src/main.rs b/src/main.rs index 6e29c85948..1b0f7f3dcc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,19 +1,19 @@ use nu_cli::{create_default_context, report_parsing_error, report_shell_error, NuHighlighter}; use nu_engine::eval_block; -use nu_parser::{ParserState, ParserWorkingSet}; +use nu_protocol::{EngineState, StateWorkingSet}; #[cfg(test)] mod tests; fn main() -> std::io::Result<()> { - let parser_state = create_default_context(); + let engine_state = create_default_context(); if let Some(path) = std::env::args().nth(1) { let file = std::fs::read(&path)?; let (block, delta) = { - let parser_state = parser_state.borrow(); - let mut working_set = ParserWorkingSet::new(&*parser_state); + let engine_state = engine_state.borrow(); + let mut working_set = StateWorkingSet::new(&*engine_state); let (output, err) = working_set.parse_file(&path, &file, false); if let Some(err) = err { let _ = report_parsing_error(&working_set, &err); @@ -23,10 +23,10 @@ fn main() -> std::io::Result<()> { (output, working_set.render()) }; - ParserState::merge_delta(&mut *parser_state.borrow_mut(), delta); + EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta); let state = nu_engine::State { - parser_state: parser_state.clone(), + engine_state: engine_state.clone(), stack: nu_engine::Stack::new(), }; @@ -35,8 +35,8 @@ fn main() -> std::io::Result<()> { println!("{}", value.into_string()); } Err(err) => { - let parser_state = parser_state.borrow(); - let working_set = ParserWorkingSet::new(&*parser_state); + let engine_state = engine_state.borrow(); + let working_set = StateWorkingSet::new(&*engine_state); let _ = report_shell_error(&working_set, &err); @@ -54,7 +54,7 @@ fn main() -> std::io::Result<()> { "history.txt".into(), )?))? .with_highlighter(Box::new(NuHighlighter { - parser_state: parser_state.clone(), + engine_state: engine_state.clone(), })); let prompt = DefaultPrompt::new(1); @@ -71,8 +71,8 @@ fn main() -> std::io::Result<()> { // println!("input: '{}'", s); let (block, delta) = { - let parser_state = parser_state.borrow(); - let mut working_set = ParserWorkingSet::new(&*parser_state); + let engine_state = engine_state.borrow(); + let mut working_set = StateWorkingSet::new(&*engine_state); let (output, err) = working_set.parse_file( &format!("line_{}", current_line), s.as_bytes(), @@ -85,10 +85,10 @@ fn main() -> std::io::Result<()> { (output, working_set.render()) }; - ParserState::merge_delta(&mut *parser_state.borrow_mut(), delta); + EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta); let state = nu_engine::State { - parser_state: parser_state.clone(), + engine_state: engine_state.clone(), stack: stack.clone(), }; @@ -97,8 +97,8 @@ fn main() -> std::io::Result<()> { println!("{}", value.into_string()); } Err(err) => { - let parser_state = parser_state.borrow(); - let working_set = ParserWorkingSet::new(&*parser_state); + let engine_state = engine_state.borrow(); + let working_set = StateWorkingSet::new(&*engine_state); let _ = report_shell_error(&working_set, &err); } From 94687a76030e4fe749c9d98198af6d4876b6c6e9 Mon Sep 17 00:00:00 2001 From: JT Date: Fri, 3 Sep 2021 06:21:37 +1200 Subject: [PATCH 04/13] Back to working state --- crates/nu-cli/src/default_context.rs | 39 +++++++++--------- crates/nu-cli/src/errors.rs | 2 +- crates/nu-cli/src/syntax_highlight.rs | 8 ++-- crates/nu-engine/src/eval.rs | 38 +++++++++--------- crates/nu-engine/src/lib.rs | 6 --- crates/nu-parser/src/errors.rs | 2 +- crates/nu-parser/src/flatten.rs | 4 +- crates/nu-parser/src/lib.rs | 4 +- crates/nu-parser/src/parser.rs | 5 ++- crates/nu-parser/src/type_check.rs | 6 ++- crates/nu-parser/tests/test_parser.rs | 40 ++++++++++--------- crates/nu-protocol/src/{ => ast}/block.rs | 2 +- crates/nu-protocol/src/{ => ast}/call.rs | 3 +- crates/nu-protocol/src/{ => ast}/expr.rs | 3 +- .../nu-protocol/src/{ => ast}/expression.rs | 3 +- crates/nu-protocol/src/ast/mod.rs | 15 +++++++ crates/nu-protocol/src/{ => ast}/operator.rs | 0 crates/nu-protocol/src/{ => ast}/pipeline.rs | 2 +- crates/nu-protocol/src/{ => ast}/statement.rs | 3 +- crates/nu-protocol/src/engine/call_info.rs | 8 ++++ .../nu-protocol/src/{ => engine}/command.rs | 12 ++++-- crates/nu-protocol/src/engine/command_args.rs | 7 ++++ .../src/{ => engine}/engine_state.rs | 3 +- .../src/engine/evaluation_context.rs} | 10 ++--- crates/nu-protocol/src/engine/mod.rs | 11 +++++ crates/nu-protocol/src/lib.rs | 20 +--------- crates/nu-protocol/src/signature.rs | 14 ++++++- src/main.rs | 16 ++++---- 28 files changed, 170 insertions(+), 116 deletions(-) rename crates/nu-protocol/src/{ => ast}/block.rs (97%) rename crates/nu-protocol/src/{ => ast}/call.rs (90%) rename crates/nu-protocol/src/{ => ast}/expr.rs (86%) rename crates/nu-protocol/src/{ => ast}/expression.rs (96%) create mode 100644 crates/nu-protocol/src/ast/mod.rs rename crates/nu-protocol/src/{ => ast}/operator.rs (100%) rename crates/nu-protocol/src/{ => ast}/pipeline.rs (90%) rename crates/nu-protocol/src/{ => ast}/statement.rs (69%) create mode 100644 crates/nu-protocol/src/engine/call_info.rs rename crates/nu-protocol/src/{ => engine}/command.rs (79%) create mode 100644 crates/nu-protocol/src/engine/command_args.rs rename crates/nu-protocol/src/{ => engine}/engine_state.rs (99%) rename crates/{nu-engine/src/state.rs => nu-protocol/src/engine/evaluation_context.rs} (93%) create mode 100644 crates/nu-protocol/src/engine/mod.rs diff --git a/crates/nu-cli/src/default_context.rs b/crates/nu-cli/src/default_context.rs index 4e349ca52c..2c59c6748e 100644 --- a/crates/nu-cli/src/default_context.rs +++ b/crates/nu-cli/src/default_context.rs @@ -1,6 +1,9 @@ use std::{cell::RefCell, rc::Rc}; -use nu_protocol::{EngineState, Signature, StateWorkingSet, SyntaxShape}; +use nu_protocol::{ + engine::{EngineState, StateWorkingSet}, + Signature, SyntaxShape, +}; pub fn create_default_context() -> Rc> { let engine_state = Rc::new(RefCell::new(EngineState::new())); @@ -10,7 +13,7 @@ pub fn create_default_context() -> Rc> { let sig = Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("if") .required("cond", SyntaxShape::Expression, "condition") @@ -20,7 +23,7 @@ pub fn create_default_context() -> Rc> { SyntaxShape::Keyword(b"else".to_vec(), Box::new(SyntaxShape::Expression)), "optional else followed by else block", ); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("let") .required("var_name", SyntaxShape::VarWithOptType, "variable name") @@ -29,7 +32,7 @@ pub fn create_default_context() -> Rc> { SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), "equals sign followed by value", ); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("let-env") .required("var_name", SyntaxShape::String, "variable name") @@ -38,7 +41,7 @@ pub fn create_default_context() -> Rc> { SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::String)), "equals sign followed by value", ); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("alias") .required("name", SyntaxShape::String, "name of the alias") @@ -47,16 +50,16 @@ pub fn create_default_context() -> Rc> { SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), "equals sign followed by value", ); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("build-string").rest(SyntaxShape::String, "list of string"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("def") .required("def_name", SyntaxShape::String, "definition name") .required("params", SyntaxShape::Signature, "parameters") .required("block", SyntaxShape::Block, "body of the definition"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("for") .required( @@ -70,11 +73,11 @@ pub fn create_default_context() -> Rc> { "range of the loop", ) .required("block", SyntaxShape::Block, "the block to run"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("benchmark").required("block", SyntaxShape::Block, "the block to run"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); // let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); // working_set.add_decl(sig.into()); @@ -84,25 +87,25 @@ pub fn create_default_context() -> Rc> { // .switch("--rock", "rock!!", Some('r')); // working_set.add_decl(sig.into()); let sig = Signature::build("exit"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("vars"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("decls"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("blocks"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("stack"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("add"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("add it"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); let sig = Signature::build("add it together") .required("x", SyntaxShape::Int, "x value") .required("y", SyntaxShape::Int, "y value"); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); working_set.render() }; diff --git a/crates/nu-cli/src/errors.rs b/crates/nu-cli/src/errors.rs index 7115dc174a..7c620c9b9c 100644 --- a/crates/nu-cli/src/errors.rs +++ b/crates/nu-cli/src/errors.rs @@ -3,7 +3,7 @@ use core::ops::Range; use codespan_reporting::diagnostic::{Diagnostic, Label}; use codespan_reporting::term::termcolor::{ColorChoice, StandardStream}; use nu_parser::ParseError; -use nu_protocol::{ShellError, Span, StateWorkingSet}; +use nu_protocol::{engine::StateWorkingSet, ShellError, Span}; fn convert_span_to_diag( working_set: &StateWorkingSet, diff --git a/crates/nu-cli/src/syntax_highlight.rs b/crates/nu-cli/src/syntax_highlight.rs index ac58ff9068..9a57dd71ed 100644 --- a/crates/nu-cli/src/syntax_highlight.rs +++ b/crates/nu-cli/src/syntax_highlight.rs @@ -1,6 +1,6 @@ use nu_ansi_term::Style; -use nu_parser::FlatShape; -use nu_protocol::{EngineState, StateWorkingSet}; +use nu_parser::{flatten_block, parse_source, FlatShape}; +use nu_protocol::engine::{EngineState, StateWorkingSet}; use reedline::{Highlighter, StyledText}; use std::{cell::RefCell, rc::Rc}; @@ -13,9 +13,9 @@ impl Highlighter for NuHighlighter { let (shapes, global_span_offset) = { let engine_state = self.engine_state.borrow(); let mut working_set = StateWorkingSet::new(&*engine_state); - let (block, _) = working_set.parse_source(line.as_bytes(), false); + let (block, _) = parse_source(&mut working_set, line.as_bytes(), false); - let shapes = working_set.flatten_block(&block); + let shapes = flatten_block(&working_set, &block); (shapes, engine_state.next_span_start()) }; diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index 5013c39955..f173c19f5c 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -1,7 +1,7 @@ use std::time::Instant; -use crate::state::State; -use nu_protocol::{Block, Call, Expr, Expression, Operator, Statement}; +use nu_protocol::ast::{Block, Call, Expr, Expression, Operator, Statement}; +use nu_protocol::engine::EvaluationContext; use nu_protocol::{IntoRowStream, IntoValueStream, ShellError, Span, Value}; pub fn eval_operator(op: &Expression) -> Result { @@ -14,16 +14,16 @@ pub fn eval_operator(op: &Expression) -> Result { } } -fn eval_call(state: &State, call: &Call) -> Result { +fn eval_call(state: &EvaluationContext, call: &Call) -> Result { let engine_state = state.engine_state.borrow(); let decl = engine_state.get_decl(call.decl_id); - if let Some(block_id) = decl.body { + if let Some(block_id) = decl.get_custom_command() { let state = state.enter_scope(); for (arg, param) in call.positional.iter().zip( - decl.signature + decl.signature() .required_positional .iter() - .chain(decl.signature.optional_positional.iter()), + .chain(decl.signature().optional_positional.iter()), ) { let result = eval_expression(&state, arg)?; let var_id = param @@ -35,7 +35,7 @@ fn eval_call(state: &State, call: &Call) -> Result { let engine_state = state.engine_state.borrow(); let block = engine_state.get_block(block_id); eval_block(&state, block) - } else if decl.signature.name == "let" { + } else if decl.signature().name == "let" { let var_id = call.positional[0] .as_var() .expect("internal error: missing variable"); @@ -52,7 +52,7 @@ fn eval_call(state: &State, call: &Call) -> Result { Ok(Value::Nothing { span: call.positional[0].span, }) - } else if decl.signature.name == "let-env" { + } else if decl.signature().name == "let-env" { let env_var = call.positional[0] .as_string() .expect("internal error: missing variable"); @@ -70,7 +70,7 @@ fn eval_call(state: &State, call: &Call) -> Result { Ok(Value::Nothing { span: call.positional[0].span, }) - } else if decl.signature.name == "if" { + } else if decl.signature().name == "if" { let cond = &call.positional[0]; let then_block = call.positional[1] .as_block() @@ -103,7 +103,7 @@ fn eval_call(state: &State, call: &Call) -> Result { } _ => Err(ShellError::CantConvert("bool".into(), result.span())), } - } else if decl.signature.name == "build-string" { + } else if decl.signature().name == "build-string" { let mut output = vec![]; for expr in &call.positional { @@ -115,7 +115,7 @@ fn eval_call(state: &State, call: &Call) -> Result { val: output.join(""), span: call.head, }) - } else if decl.signature.name == "benchmark" { + } else if decl.signature().name == "benchmark" { let block = call.positional[0] .as_block() .expect("internal error: expected block"); @@ -130,7 +130,7 @@ fn eval_call(state: &State, call: &Call) -> Result { Ok(Value::Nothing { span: call.positional[0].span, }) - } else if decl.signature.name == "for" { + } else if decl.signature().name == "for" { let var_id = call.positional[0] .as_var() .expect("internal error: missing variable"); @@ -167,26 +167,26 @@ fn eval_call(state: &State, call: &Call) -> Result { Ok(Value::Nothing { span: call.positional[0].span, }) - } else if decl.signature.name == "vars" { + } else if decl.signature().name == "vars" { state.engine_state.borrow().print_vars(); Ok(Value::Nothing { span: call.head }) - } else if decl.signature.name == "decls" { + } else if decl.signature().name == "decls" { state.engine_state.borrow().print_decls(); Ok(Value::Nothing { span: call.head }) - } else if decl.signature.name == "blocks" { + } else if decl.signature().name == "blocks" { state.engine_state.borrow().print_blocks(); Ok(Value::Nothing { span: call.head }) - } else if decl.signature.name == "stack" { + } else if decl.signature().name == "stack" { state.print_stack(); Ok(Value::Nothing { span: call.head }) - } else if decl.signature.name == "def" || decl.signature.name == "alias" { + } else if decl.signature().name == "def" || decl.signature().name == "alias" { Ok(Value::Nothing { span: call.head }) } else { Err(ShellError::Unsupported(call.head)) } } -pub fn eval_expression(state: &State, expr: &Expression) -> Result { +pub fn eval_expression(state: &EvaluationContext, expr: &Expression) -> Result { match &expr.expr { Expr::Bool(b) => Ok(Value::Bool { val: *b, @@ -278,7 +278,7 @@ pub fn eval_expression(state: &State, expr: &Expression) -> Result Result { +pub fn eval_block(state: &EvaluationContext, block: &Block) -> Result { let mut last = Ok(Value::Nothing { span: Span { start: 0, end: 0 }, }); diff --git a/crates/nu-engine/src/lib.rs b/crates/nu-engine/src/lib.rs index 4143f4ae64..c912ee4772 100644 --- a/crates/nu-engine/src/lib.rs +++ b/crates/nu-engine/src/lib.rs @@ -1,9 +1,3 @@ -mod command; mod eval; -mod example; -mod state; -pub use command::Command; pub use eval::{eval_block, eval_expression, eval_operator}; -pub use example::Example; -pub use state::{Stack, State}; diff --git a/crates/nu-parser/src/errors.rs b/crates/nu-parser/src/errors.rs index db82f95008..5f16a34711 100644 --- a/crates/nu-parser/src/errors.rs +++ b/crates/nu-parser/src/errors.rs @@ -1,4 +1,4 @@ -use nu_protocol::{Span, StateWorkingSet, Type}; +use nu_protocol::{engine::StateWorkingSet, Span, Type}; use std::ops::Range; #[derive(Debug)] diff --git a/crates/nu-parser/src/flatten.rs b/crates/nu-parser/src/flatten.rs index dea7795c5b..a2b490bf82 100644 --- a/crates/nu-parser/src/flatten.rs +++ b/crates/nu-parser/src/flatten.rs @@ -1,5 +1,5 @@ -use nu_protocol::Span; -use nu_protocol::{Block, Expr, Expression, Pipeline, StateWorkingSet, Statement}; +use nu_protocol::ast::{Block, Expr, Expression, Pipeline, Statement}; +use nu_protocol::{engine::StateWorkingSet, Span}; #[derive(Debug)] pub enum FlatShape { diff --git a/crates/nu-parser/src/lib.rs b/crates/nu-parser/src/lib.rs index a15f2dfb42..fb2d2e8e2e 100644 --- a/crates/nu-parser/src/lib.rs +++ b/crates/nu-parser/src/lib.rs @@ -6,7 +6,7 @@ mod parser; mod type_check; pub use errors::ParseError; -pub use flatten::FlatShape; +pub use flatten::{flatten_block, FlatShape}; pub use lex::{lex, Token, TokenContents}; pub use lite_parse::{lite_parse, LiteBlock}; -pub use parser::{Import, VarDecl}; +pub use parser::{parse_file, parse_source, Import, VarDecl}; diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index 0e1afd1ef4..12e6584df8 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -5,8 +5,9 @@ use crate::{ }; use nu_protocol::{ - span, Block, BlockId, Call, DeclId, Expr, Expression, Flag, Operator, Pipeline, PositionalArg, - Signature, Span, StateWorkingSet, Statement, SyntaxShape, Type, VarId, + ast::{Block, Call, Expr, Expression, Operator, Pipeline, Statement}, + engine::StateWorkingSet, + span, BlockId, DeclId, Flag, PositionalArg, Signature, Span, SyntaxShape, Type, VarId, }; #[derive(Debug, Clone)] diff --git a/crates/nu-parser/src/type_check.rs b/crates/nu-parser/src/type_check.rs index 0e88dac135..9ea5f68eaf 100644 --- a/crates/nu-parser/src/type_check.rs +++ b/crates/nu-parser/src/type_check.rs @@ -1,5 +1,9 @@ use crate::ParseError; -use nu_protocol::{Expr, Expression, Operator, StateWorkingSet, Type}; +use nu_protocol::{ + ast::{Expr, Expression, Operator}, + engine::StateWorkingSet, + Type, +}; pub fn type_compatible(lhs: &Type, rhs: &Type) -> bool { match (lhs, rhs) { diff --git a/crates/nu-parser/tests/test_parser.rs b/crates/nu-parser/tests/test_parser.rs index e13075196b..959925d96a 100644 --- a/crates/nu-parser/tests/test_parser.rs +++ b/crates/nu-parser/tests/test_parser.rs @@ -1,13 +1,17 @@ use nu_parser::ParseError; use nu_parser::*; -use nu_protocol::{EngineState, Signature, SyntaxShape}; +use nu_protocol::{ + ast::{Expr, Expression, Statement}, + engine::{EngineState, StateWorkingSet}, + Signature, SyntaxShape, +}; #[test] pub fn parse_int() { let engine_state = EngineState::new(); let mut working_set = StateWorkingSet::new(&engine_state); - let (block, err) = working_set.parse_source(b"3", true); + let (block, err) = parse_source(&mut working_set, b"3", true); assert!(err.is_none()); assert!(block.len() == 1); @@ -26,9 +30,9 @@ pub fn parse_call() { let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); - let (block, err) = working_set.parse_source(b"foo", true); + let (block, err) = parse_source(&mut working_set, b"foo", true); assert!(err.is_none()); assert!(block.len() == 1); @@ -50,9 +54,9 @@ pub fn parse_call_missing_flag_arg() { let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); - let (_, err) = working_set.parse_source(b"foo --jazz", true); + let (_, err) = parse_source(&mut working_set, b"foo --jazz", true); assert!(matches!(err, Some(ParseError::MissingFlagParam(..)))); } @@ -62,9 +66,9 @@ pub fn parse_call_missing_short_flag_arg() { let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); - working_set.add_decl(sig.into()); + working_set.add_decl(sig.predeclare()); - let (_, err) = working_set.parse_source(b"foo -j", true); + let (_, err) = parse_source(&mut working_set, b"foo -j", true); assert!(matches!(err, Some(ParseError::MissingFlagParam(..)))); } @@ -76,8 +80,8 @@ pub fn parse_call_too_many_shortflag_args() { let sig = Signature::build("foo") .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')) .named("--math", SyntaxShape::Int, "math!!", Some('m')); - working_set.add_decl(sig.into()); - let (_, err) = working_set.parse_source(b"foo -mj", true); + working_set.add_decl(sig.predeclare()); + let (_, err) = parse_source(&mut working_set, b"foo -mj", true); assert!(matches!( err, Some(ParseError::ShortFlagBatchCantTakeArg(..)) @@ -90,8 +94,8 @@ pub fn parse_call_unknown_shorthand() { let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").switch("--jazz", "jazz!!", Some('j')); - working_set.add_decl(sig.into()); - let (_, err) = working_set.parse_source(b"foo -mj", true); + working_set.add_decl(sig.predeclare()); + let (_, err) = parse_source(&mut working_set, b"foo -mj", true); assert!(matches!(err, Some(ParseError::UnknownFlag(..)))); } @@ -101,8 +105,8 @@ pub fn parse_call_extra_positional() { let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").switch("--jazz", "jazz!!", Some('j')); - working_set.add_decl(sig.into()); - let (_, err) = working_set.parse_source(b"foo -j 100", true); + working_set.add_decl(sig.predeclare()); + let (_, err) = parse_source(&mut working_set, b"foo -j 100", true); assert!(matches!(err, Some(ParseError::ExtraPositional(..)))); } @@ -112,8 +116,8 @@ pub fn parse_call_missing_req_positional() { let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").required("jazz", SyntaxShape::Int, "jazz!!"); - working_set.add_decl(sig.into()); - let (_, err) = working_set.parse_source(b"foo", true); + working_set.add_decl(sig.predeclare()); + let (_, err) = parse_source(&mut working_set, b"foo", true); assert!(matches!(err, Some(ParseError::MissingPositional(..)))); } @@ -123,7 +127,7 @@ pub fn parse_call_missing_req_flag() { let mut working_set = StateWorkingSet::new(&engine_state); let sig = Signature::build("foo").required_named("--jazz", SyntaxShape::Int, "jazz!!", None); - working_set.add_decl(sig.into()); - let (_, err) = working_set.parse_source(b"foo", true); + working_set.add_decl(sig.predeclare()); + let (_, err) = parse_source(&mut working_set, b"foo", true); assert!(matches!(err, Some(ParseError::MissingRequiredFlag(..)))); } diff --git a/crates/nu-protocol/src/block.rs b/crates/nu-protocol/src/ast/block.rs similarity index 97% rename from crates/nu-protocol/src/block.rs rename to crates/nu-protocol/src/ast/block.rs index 3b2f66bef1..365d2e76c9 100644 --- a/crates/nu-protocol/src/block.rs +++ b/crates/nu-protocol/src/ast/block.rs @@ -1,6 +1,6 @@ use std::ops::{Index, IndexMut}; -use crate::Statement; +use super::Statement; #[derive(Debug, Clone)] pub struct Block { diff --git a/crates/nu-protocol/src/call.rs b/crates/nu-protocol/src/ast/call.rs similarity index 90% rename from crates/nu-protocol/src/call.rs rename to crates/nu-protocol/src/ast/call.rs index df90a12060..efe5d7b482 100644 --- a/crates/nu-protocol/src/call.rs +++ b/crates/nu-protocol/src/ast/call.rs @@ -1,4 +1,5 @@ -use crate::{DeclId, Expression, Span}; +use super::Expression; +use crate::{DeclId, Span}; #[derive(Debug, Clone)] pub struct Call { diff --git a/crates/nu-protocol/src/expr.rs b/crates/nu-protocol/src/ast/expr.rs similarity index 86% rename from crates/nu-protocol/src/expr.rs rename to crates/nu-protocol/src/ast/expr.rs index cfc74799c2..818db24cd5 100644 --- a/crates/nu-protocol/src/expr.rs +++ b/crates/nu-protocol/src/ast/expr.rs @@ -1,4 +1,5 @@ -use crate::{BlockId, Call, Expression, Operator, Signature, Span, VarId}; +use super::{Call, Expression, Operator}; +use crate::{BlockId, Signature, Span, VarId}; #[derive(Debug, Clone)] pub enum Expr { diff --git a/crates/nu-protocol/src/expression.rs b/crates/nu-protocol/src/ast/expression.rs similarity index 96% rename from crates/nu-protocol/src/expression.rs rename to crates/nu-protocol/src/ast/expression.rs index 176bb65ce4..d17b05428c 100644 --- a/crates/nu-protocol/src/expression.rs +++ b/crates/nu-protocol/src/ast/expression.rs @@ -1,4 +1,5 @@ -use crate::{BlockId, Expr, Operator, Signature, Span, Type, VarId}; +use super::{Expr, Operator}; +use crate::{BlockId, Signature, Span, Type, VarId}; #[derive(Debug, Clone)] pub struct Expression { diff --git a/crates/nu-protocol/src/ast/mod.rs b/crates/nu-protocol/src/ast/mod.rs new file mode 100644 index 0000000000..90c3901bde --- /dev/null +++ b/crates/nu-protocol/src/ast/mod.rs @@ -0,0 +1,15 @@ +mod block; +mod call; +mod expr; +mod expression; +mod operator; +mod pipeline; +mod statement; + +pub use block::*; +pub use call::*; +pub use expr::*; +pub use expression::*; +pub use operator::*; +pub use pipeline::*; +pub use statement::*; diff --git a/crates/nu-protocol/src/operator.rs b/crates/nu-protocol/src/ast/operator.rs similarity index 100% rename from crates/nu-protocol/src/operator.rs rename to crates/nu-protocol/src/ast/operator.rs diff --git a/crates/nu-protocol/src/pipeline.rs b/crates/nu-protocol/src/ast/pipeline.rs similarity index 90% rename from crates/nu-protocol/src/pipeline.rs rename to crates/nu-protocol/src/ast/pipeline.rs index cdeb722158..fd4425290c 100644 --- a/crates/nu-protocol/src/pipeline.rs +++ b/crates/nu-protocol/src/ast/pipeline.rs @@ -1,4 +1,4 @@ -use crate::Expression; +use crate::ast::Expression; #[derive(Debug, Clone)] pub struct Pipeline { diff --git a/crates/nu-protocol/src/statement.rs b/crates/nu-protocol/src/ast/statement.rs similarity index 69% rename from crates/nu-protocol/src/statement.rs rename to crates/nu-protocol/src/ast/statement.rs index 27b86f005e..24703a8cfb 100644 --- a/crates/nu-protocol/src/statement.rs +++ b/crates/nu-protocol/src/ast/statement.rs @@ -1,4 +1,5 @@ -use crate::{DeclId, Expression, Pipeline}; +use super::{Expression, Pipeline}; +use crate::DeclId; #[derive(Debug, Clone)] pub enum Statement { diff --git a/crates/nu-protocol/src/engine/call_info.rs b/crates/nu-protocol/src/engine/call_info.rs new file mode 100644 index 0000000000..a30dc2d848 --- /dev/null +++ b/crates/nu-protocol/src/engine/call_info.rs @@ -0,0 +1,8 @@ +use crate::ast::Call; +use crate::Span; + +#[derive(Debug, Clone)] +pub struct UnevaluatedCallInfo { + pub args: Call, + pub name_span: Span, +} diff --git a/crates/nu-protocol/src/command.rs b/crates/nu-protocol/src/engine/command.rs similarity index 79% rename from crates/nu-protocol/src/command.rs rename to crates/nu-protocol/src/engine/command.rs index 08caf637c4..8e8118fc8d 100644 --- a/crates/nu-protocol/src/command.rs +++ b/crates/nu-protocol/src/engine/command.rs @@ -1,4 +1,6 @@ -use crate::{Example, Signature}; +use crate::{BlockId, Example, ShellError, Signature, Value}; + +use super::CommandArgs; pub trait Command { fn name(&self) -> &str; @@ -13,6 +15,8 @@ pub trait Command { "" } + fn run(&self, args: CommandArgs) -> Result; + // fn run(&self, args: CommandArgs) -> Result { // let context = args.context.clone(); // let stream = self.run_with_actions(args)?; @@ -53,8 +57,8 @@ pub trait Command { false } - // Is a custom command i.e. def blah [] { } - fn is_custom(&self) -> bool { - false + // If command is a custom command i.e. def blah [] { }, get the block id + fn get_custom_command(&self) -> Option { + None } } diff --git a/crates/nu-protocol/src/engine/command_args.rs b/crates/nu-protocol/src/engine/command_args.rs new file mode 100644 index 0000000000..a9468e0118 --- /dev/null +++ b/crates/nu-protocol/src/engine/command_args.rs @@ -0,0 +1,7 @@ +use super::{EvaluationContext, UnevaluatedCallInfo}; + +pub struct CommandArgs { + pub context: EvaluationContext, + pub call_info: UnevaluatedCallInfo, + pub input: crate::Value, +} diff --git a/crates/nu-protocol/src/engine_state.rs b/crates/nu-protocol/src/engine/engine_state.rs similarity index 99% rename from crates/nu-protocol/src/engine_state.rs rename to crates/nu-protocol/src/engine/engine_state.rs index 8d96c9bdab..af1f940ac6 100644 --- a/crates/nu-protocol/src/engine_state.rs +++ b/crates/nu-protocol/src/engine/engine_state.rs @@ -1,4 +1,5 @@ -use crate::{Block, BlockId, Command, DeclId, Span, Type, VarId}; +use super::Command; +use crate::{ast::Block, BlockId, DeclId, Span, Type, VarId}; use core::panic; use std::{collections::HashMap, ops::Range, slice::Iter}; diff --git a/crates/nu-engine/src/state.rs b/crates/nu-protocol/src/engine/evaluation_context.rs similarity index 93% rename from crates/nu-engine/src/state.rs rename to crates/nu-protocol/src/engine/evaluation_context.rs index e9b5cff4ff..f1baa94949 100644 --- a/crates/nu-engine/src/state.rs +++ b/crates/nu-protocol/src/engine/evaluation_context.rs @@ -1,19 +1,19 @@ -use nu_protocol::EngineState; +use super::EngineState; use std::{cell::RefCell, collections::HashMap, rc::Rc}; -use nu_protocol::{ShellError, Value, VarId}; +use crate::{ShellError, Value, VarId}; -pub struct State { +pub struct EvaluationContext { pub engine_state: Rc>, pub stack: Stack, } -impl State { +impl EvaluationContext { pub fn get_var(&self, var_id: VarId) -> Result { self.stack.get_var(var_id) } - pub fn enter_scope(&self) -> State { + pub fn enter_scope(&self) -> EvaluationContext { Self { engine_state: self.engine_state.clone(), stack: self.stack.clone().enter_scope(), diff --git a/crates/nu-protocol/src/engine/mod.rs b/crates/nu-protocol/src/engine/mod.rs new file mode 100644 index 0000000000..68ffac9616 --- /dev/null +++ b/crates/nu-protocol/src/engine/mod.rs @@ -0,0 +1,11 @@ +mod call_info; +mod command; +mod command_args; +mod engine_state; +mod evaluation_context; + +pub use call_info::*; +pub use command::*; +pub use command_args::*; +pub use engine_state::*; +pub use evaluation_context::*; diff --git a/crates/nu-protocol/src/lib.rs b/crates/nu-protocol/src/lib.rs index d4e1166cf4..3e7f61639c 100644 --- a/crates/nu-protocol/src/lib.rs +++ b/crates/nu-protocol/src/lib.rs @@ -1,35 +1,19 @@ -mod block; -mod call; -mod command; -mod engine_state; +pub mod ast; +pub mod engine; mod example; -mod expr; -mod expression; mod id; -mod operator; -mod pipeline; mod shell_error; mod signature; mod span; -mod statement; mod syntax_shape; mod ty; mod value; -pub use block::*; -pub use call::*; -pub use command::*; -pub use engine_state::*; pub use example::*; -pub use expr::*; -pub use expression::*; pub use id::*; -pub use operator::*; -pub use pipeline::*; pub use shell_error::*; pub use signature::*; pub use span::*; -pub use statement::*; pub use syntax_shape::*; pub use ty::*; pub use value::*; diff --git a/crates/nu-protocol/src/signature.rs b/crates/nu-protocol/src/signature.rs index f5fc981223..a797ecbf4b 100644 --- a/crates/nu-protocol/src/signature.rs +++ b/crates/nu-protocol/src/signature.rs @@ -1,5 +1,5 @@ +use crate::engine::Command; use crate::BlockId; -use crate::Command; use crate::SyntaxShape; use crate::VarId; @@ -314,6 +314,10 @@ impl Command for Predeclaration { fn usage(&self) -> &str { &self.signature.usage } + + fn run(&self, _args: crate::engine::CommandArgs) -> Result { + panic!("Internal error: can't run a predeclaration without a body") + } } struct BlockCommand { @@ -333,4 +337,12 @@ impl Command for BlockCommand { fn usage(&self) -> &str { &self.signature.usage } + + fn run(&self, _args: crate::engine::CommandArgs) -> Result { + panic!("Internal error: can't run custom command with 'run', use block_id"); + } + + fn get_custom_command(&self) -> Option { + Some(self.block_id) + } } diff --git a/src/main.rs b/src/main.rs index 1b0f7f3dcc..030386a055 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ use nu_cli::{create_default_context, report_parsing_error, report_shell_error, NuHighlighter}; use nu_engine::eval_block; -use nu_protocol::{EngineState, StateWorkingSet}; +use nu_parser::parse_file; +use nu_protocol::engine::{EngineState, EvaluationContext, StateWorkingSet}; #[cfg(test)] mod tests; @@ -14,7 +15,7 @@ fn main() -> std::io::Result<()> { let (block, delta) = { let engine_state = engine_state.borrow(); let mut working_set = StateWorkingSet::new(&*engine_state); - let (output, err) = working_set.parse_file(&path, &file, false); + let (output, err) = parse_file(&mut working_set, &path, &file, false); if let Some(err) = err { let _ = report_parsing_error(&working_set, &err); @@ -25,9 +26,9 @@ fn main() -> std::io::Result<()> { EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta); - let state = nu_engine::State { + let state = EvaluationContext { engine_state: engine_state.clone(), - stack: nu_engine::Stack::new(), + stack: nu_protocol::engine::Stack::new(), }; match eval_block(&state, &block) { @@ -59,7 +60,7 @@ fn main() -> std::io::Result<()> { let prompt = DefaultPrompt::new(1); let mut current_line = 1; - let stack = nu_engine::Stack::new(); + let stack = nu_protocol::engine::Stack::new(); loop { let input = line_editor.read_line(&prompt); @@ -73,7 +74,8 @@ fn main() -> std::io::Result<()> { let (block, delta) = { let engine_state = engine_state.borrow(); let mut working_set = StateWorkingSet::new(&*engine_state); - let (output, err) = working_set.parse_file( + let (output, err) = parse_file( + &mut working_set, &format!("line_{}", current_line), s.as_bytes(), false, @@ -87,7 +89,7 @@ fn main() -> std::io::Result<()> { EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta); - let state = nu_engine::State { + let state = nu_protocol::engine::EvaluationContext { engine_state: engine_state.clone(), stack: stack.clone(), }; From 7c8504ea24587aa03a4c363a1e3a77fe998243ae Mon Sep 17 00:00:00 2001 From: JT Date: Fri, 3 Sep 2021 10:58:15 +1200 Subject: [PATCH 05/13] Add commands --- Cargo.lock | 5 + Cargo.toml | 1 + crates/nu-cli/src/default_context.rs | 118 ----------- crates/nu-cli/src/lib.rs | 2 - crates/nu-command/Cargo.toml | 2 + crates/nu-command/src/alias.rs | 34 ++++ crates/nu-command/src/benchmark.rs | 44 +++++ crates/nu-command/src/build_string.rs | 39 ++++ crates/nu-command/src/def.rs | 31 +++ crates/nu-command/src/default_context.rs | 55 ++++++ crates/nu-command/src/for_.rs | 75 +++++++ crates/nu-command/src/if_.rs | 67 +++++++ crates/nu-command/src/let_.rs | 50 +++++ crates/nu-command/src/let_env.rs | 51 +++++ crates/nu-command/src/lib.rs | 26 ++- crates/nu-engine/src/eval.rs | 187 +++--------------- crates/nu-parser/src/errors.rs | 3 +- crates/nu-parser/src/parser.rs | 5 +- crates/nu-protocol/src/engine/command.rs | 23 +-- crates/nu-protocol/src/engine/command_args.rs | 7 - crates/nu-protocol/src/engine/mod.rs | 2 - crates/nu-protocol/src/signature.rs | 17 +- src/main.rs | 16 +- 23 files changed, 536 insertions(+), 324 deletions(-) delete mode 100644 crates/nu-cli/src/default_context.rs create mode 100644 crates/nu-command/src/alias.rs create mode 100644 crates/nu-command/src/benchmark.rs create mode 100644 crates/nu-command/src/build_string.rs create mode 100644 crates/nu-command/src/def.rs create mode 100644 crates/nu-command/src/default_context.rs create mode 100644 crates/nu-command/src/for_.rs create mode 100644 crates/nu-command/src/if_.rs create mode 100644 crates/nu-command/src/let_.rs create mode 100644 crates/nu-command/src/let_env.rs delete mode 100644 crates/nu-protocol/src/engine/command_args.rs diff --git a/Cargo.lock b/Cargo.lock index 2112588bf0..3c54213f68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -162,6 +162,7 @@ dependencies = [ "assert_cmd", "codespan-reporting", "nu-cli", + "nu-command", "nu-engine", "nu-parser", "nu-protocol", @@ -300,6 +301,10 @@ dependencies = [ [[package]] name = "nu-command" version = "0.1.0" +dependencies = [ + "nu-engine", + "nu-protocol", +] [[package]] name = "nu-engine" diff --git a/Cargo.toml b/Cargo.toml index 6f1ff009f8..225d8c4f8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = ["crates/nu-cli", "crates/nu-engine", "crates/nu-parser", "crates/nu-c reedline = { git = "https://github.com/jntrnr/reedline", branch = "main" } codespan-reporting = "0.11.1" nu-cli = { path="./crates/nu-cli" } +nu-command = { path="./crates/nu-command" } nu-engine = { path="./crates/nu-engine" } nu-parser = { path="./crates/nu-parser" } nu-protocol = { path = "./crates/nu-protocol" } diff --git a/crates/nu-cli/src/default_context.rs b/crates/nu-cli/src/default_context.rs deleted file mode 100644 index 2c59c6748e..0000000000 --- a/crates/nu-cli/src/default_context.rs +++ /dev/null @@ -1,118 +0,0 @@ -use std::{cell::RefCell, rc::Rc}; - -use nu_protocol::{ - engine::{EngineState, StateWorkingSet}, - Signature, SyntaxShape, -}; - -pub fn create_default_context() -> Rc> { - let engine_state = Rc::new(RefCell::new(EngineState::new())); - let delta = { - let engine_state = engine_state.borrow(); - let mut working_set = StateWorkingSet::new(&*engine_state); - - let sig = - Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition"); - working_set.add_decl(sig.predeclare()); - - let sig = Signature::build("if") - .required("cond", SyntaxShape::Expression, "condition") - .required("then_block", SyntaxShape::Block, "then block") - .optional( - "else", - SyntaxShape::Keyword(b"else".to_vec(), Box::new(SyntaxShape::Expression)), - "optional else followed by else block", - ); - working_set.add_decl(sig.predeclare()); - - let sig = Signature::build("let") - .required("var_name", SyntaxShape::VarWithOptType, "variable name") - .required( - "initial_value", - SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), - "equals sign followed by value", - ); - working_set.add_decl(sig.predeclare()); - - let sig = Signature::build("let-env") - .required("var_name", SyntaxShape::String, "variable name") - .required( - "initial_value", - SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::String)), - "equals sign followed by value", - ); - working_set.add_decl(sig.predeclare()); - - let sig = Signature::build("alias") - .required("name", SyntaxShape::String, "name of the alias") - .required( - "initial_value", - SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), - "equals sign followed by value", - ); - working_set.add_decl(sig.predeclare()); - - let sig = Signature::build("build-string").rest(SyntaxShape::String, "list of string"); - working_set.add_decl(sig.predeclare()); - - let sig = Signature::build("def") - .required("def_name", SyntaxShape::String, "definition name") - .required("params", SyntaxShape::Signature, "parameters") - .required("block", SyntaxShape::Block, "body of the definition"); - working_set.add_decl(sig.predeclare()); - - let sig = Signature::build("for") - .required( - "var_name", - SyntaxShape::Variable, - "name of the looping variable", - ) - .required( - "range", - SyntaxShape::Keyword(b"in".to_vec(), Box::new(SyntaxShape::Int)), - "range of the loop", - ) - .required("block", SyntaxShape::Block, "the block to run"); - working_set.add_decl(sig.predeclare()); - - let sig = - Signature::build("benchmark").required("block", SyntaxShape::Block, "the block to run"); - working_set.add_decl(sig.predeclare()); - - // let sig = Signature::build("foo").named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')); - // working_set.add_decl(sig.into()); - - // let sig = Signature::build("bar") - // .named("--jazz", SyntaxShape::Int, "jazz!!", Some('j')) - // .switch("--rock", "rock!!", Some('r')); - // working_set.add_decl(sig.into()); - let sig = Signature::build("exit"); - working_set.add_decl(sig.predeclare()); - let sig = Signature::build("vars"); - working_set.add_decl(sig.predeclare()); - let sig = Signature::build("decls"); - working_set.add_decl(sig.predeclare()); - let sig = Signature::build("blocks"); - working_set.add_decl(sig.predeclare()); - let sig = Signature::build("stack"); - working_set.add_decl(sig.predeclare()); - - let sig = Signature::build("add"); - working_set.add_decl(sig.predeclare()); - let sig = Signature::build("add it"); - working_set.add_decl(sig.predeclare()); - - let sig = Signature::build("add it together") - .required("x", SyntaxShape::Int, "x value") - .required("y", SyntaxShape::Int, "y value"); - working_set.add_decl(sig.predeclare()); - - working_set.render() - }; - - { - EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta); - } - - engine_state -} diff --git a/crates/nu-cli/src/lib.rs b/crates/nu-cli/src/lib.rs index 7c2e42b92d..a8c602012b 100644 --- a/crates/nu-cli/src/lib.rs +++ b/crates/nu-cli/src/lib.rs @@ -1,7 +1,5 @@ -mod default_context; mod errors; mod syntax_highlight; -pub use default_context::create_default_context; pub use errors::{report_parsing_error, report_shell_error}; pub use syntax_highlight::NuHighlighter; diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index eaaba9db15..6cc6ae2f63 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -6,3 +6,5 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +nu-protocol = { path = "../nu-protocol" } +nu-engine = { path = "../nu-engine" } \ No newline at end of file diff --git a/crates/nu-command/src/alias.rs b/crates/nu-command/src/alias.rs new file mode 100644 index 0000000000..91beec2fda --- /dev/null +++ b/crates/nu-command/src/alias.rs @@ -0,0 +1,34 @@ +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct Alias; + +impl Command for Alias { + fn name(&self) -> &str { + "alias" + } + + fn usage(&self) -> &str { + "Alias a command (with optional flags) to a new name" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("alias") + .required("name", SyntaxShape::String, "name of the alias") + .required( + "initial_value", + SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), + "equals sign followed by value", + ) + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + Ok(Value::Nothing { span: call.head }) + } +} diff --git a/crates/nu-command/src/benchmark.rs b/crates/nu-command/src/benchmark.rs new file mode 100644 index 0000000000..f494da794e --- /dev/null +++ b/crates/nu-command/src/benchmark.rs @@ -0,0 +1,44 @@ +use std::time::Instant; + +use nu_engine::eval_block; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct Benchmark; + +impl Command for Benchmark { + fn name(&self) -> &str { + "benchmark" + } + + fn usage(&self) -> &str { + "Time the running time of a block" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("benchmark").required("block", SyntaxShape::Block, "the block to run") + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let block = call.positional[0] + .as_block() + .expect("internal error: expected block"); + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block); + + let state = context.enter_scope(); + let start_time = Instant::now(); + eval_block(&state, block)?; + let end_time = Instant::now(); + println!("{} ms", (end_time - start_time).as_millis()); + Ok(Value::Nothing { + span: call.positional[0].span, + }) + } +} diff --git a/crates/nu-command/src/build_string.rs b/crates/nu-command/src/build_string.rs new file mode 100644 index 0000000000..1d6d5e51f5 --- /dev/null +++ b/crates/nu-command/src/build_string.rs @@ -0,0 +1,39 @@ +use nu_engine::eval_expression; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct BuildString; + +impl Command for BuildString { + fn name(&self) -> &str { + "build-string" + } + + fn usage(&self) -> &str { + "Create a string from the arguments." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("build-string").rest(SyntaxShape::String, "list of string") + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let mut output = vec![]; + + for expr in &call.positional { + let val = eval_expression(context, expr)?; + + output.push(val.into_string()); + } + Ok(Value::String { + val: output.join(""), + span: call.head, + }) + } +} diff --git a/crates/nu-command/src/def.rs b/crates/nu-command/src/def.rs new file mode 100644 index 0000000000..25004d800d --- /dev/null +++ b/crates/nu-command/src/def.rs @@ -0,0 +1,31 @@ +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct Def; + +impl Command for Def { + fn name(&self) -> &str { + "def" + } + + fn usage(&self) -> &str { + "Define a custom command" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("def") + .required("def_name", SyntaxShape::String, "definition name") + .required("params", SyntaxShape::Signature, "parameters") + .required("block", SyntaxShape::Block, "body of the definition") + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + Ok(Value::Nothing { span: call.head }) + } +} diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs new file mode 100644 index 0000000000..f1bb536ff0 --- /dev/null +++ b/crates/nu-command/src/default_context.rs @@ -0,0 +1,55 @@ +use std::{cell::RefCell, rc::Rc}; + +use nu_protocol::{ + engine::{EngineState, StateWorkingSet}, + Signature, SyntaxShape, +}; + +use crate::{Alias, Benchmark, BuildString, Def, For, If, Let, LetEnv}; + +pub fn create_default_context() -> Rc> { + let engine_state = Rc::new(RefCell::new(EngineState::new())); + let delta = { + let engine_state = engine_state.borrow(); + let mut working_set = StateWorkingSet::new(&*engine_state); + + let sig = + Signature::build("where").required("cond", SyntaxShape::RowCondition, "condition"); + working_set.add_decl(sig.predeclare()); + + working_set.add_decl(Box::new(If)); + + working_set.add_decl(Box::new(Let)); + + working_set.add_decl(Box::new(LetEnv)); + + working_set.add_decl(Box::new(Alias)); + + working_set.add_decl(Box::new(BuildString)); + + working_set.add_decl(Box::new(Def)); + + working_set.add_decl(Box::new(For)); + + working_set.add_decl(Box::new(Benchmark)); + + let sig = Signature::build("exit"); + working_set.add_decl(sig.predeclare()); + let sig = Signature::build("vars"); + working_set.add_decl(sig.predeclare()); + let sig = Signature::build("decls"); + working_set.add_decl(sig.predeclare()); + let sig = Signature::build("blocks"); + working_set.add_decl(sig.predeclare()); + let sig = Signature::build("stack"); + working_set.add_decl(sig.predeclare()); + + working_set.render() + }; + + { + EngineState::merge_delta(&mut *engine_state.borrow_mut(), delta); + } + + engine_state +} diff --git a/crates/nu-command/src/for_.rs b/crates/nu-command/src/for_.rs new file mode 100644 index 0000000000..8eaf393e72 --- /dev/null +++ b/crates/nu-command/src/for_.rs @@ -0,0 +1,75 @@ +use nu_engine::{eval_block, eval_expression}; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, Span, SyntaxShape, Value}; + +pub struct For; + +impl Command for For { + fn name(&self) -> &str { + "for" + } + + fn usage(&self) -> &str { + "Loop over a range" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("for") + .required( + "var_name", + SyntaxShape::Variable, + "name of the looping variable", + ) + .required( + "range", + SyntaxShape::Keyword(b"in".to_vec(), Box::new(SyntaxShape::Int)), + "range of the loop", + ) + .required("block", SyntaxShape::Block, "the block to run") + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let var_id = call.positional[0] + .as_var() + .expect("internal error: missing variable"); + + let keyword_expr = call.positional[1] + .as_keyword() + .expect("internal error: missing keyword"); + let end_val = eval_expression(context, keyword_expr)?; + + let block = call.positional[2] + .as_block() + .expect("internal error: expected block"); + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block); + + let state = context.enter_scope(); + + let mut x = Value::Int { + val: 0, + span: Span::unknown(), + }; + + loop { + if x == end_val { + break; + } else { + state.add_var(var_id, x.clone()); + eval_block(&state, block)?; + } + if let Value::Int { ref mut val, .. } = x { + *val += 1 + } + } + Ok(Value::Nothing { + span: call.positional[0].span, + }) + } +} diff --git a/crates/nu-command/src/if_.rs b/crates/nu-command/src/if_.rs new file mode 100644 index 0000000000..4124dbc7b8 --- /dev/null +++ b/crates/nu-command/src/if_.rs @@ -0,0 +1,67 @@ +use nu_engine::{eval_block, eval_expression}; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{ShellError, Signature, SyntaxShape, Value}; + +pub struct If; + +impl Command for If { + fn name(&self) -> &str { + "if" + } + + fn usage(&self) -> &str { + "Create a variable and give it a value." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("if") + .required("cond", SyntaxShape::Expression, "condition") + .required("then_block", SyntaxShape::Block, "then block") + .optional( + "else", + SyntaxShape::Keyword(b"else".to_vec(), Box::new(SyntaxShape::Expression)), + "optional else followed by else block", + ) + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let cond = &call.positional[0]; + let then_block = call.positional[1] + .as_block() + .expect("internal error: expected block"); + let else_case = call.positional.get(2); + + let result = eval_expression(context, cond)?; + match result { + Value::Bool { val, span } => { + let engine_state = context.engine_state.borrow(); + if val { + let block = engine_state.get_block(then_block); + let state = context.enter_scope(); + eval_block(&state, block) + } else if let Some(else_case) = else_case { + if let Some(else_expr) = else_case.as_keyword() { + if let Some(block_id) = else_expr.as_block() { + let block = engine_state.get_block(block_id); + let state = context.enter_scope(); + eval_block(&state, block) + } else { + eval_expression(context, else_expr) + } + } else { + eval_expression(context, else_case) + } + } else { + Ok(Value::Nothing { span }) + } + } + _ => Err(ShellError::CantConvert("bool".into(), result.span())), + } + } +} diff --git a/crates/nu-command/src/let_.rs b/crates/nu-command/src/let_.rs new file mode 100644 index 0000000000..6e3a2d2fd3 --- /dev/null +++ b/crates/nu-command/src/let_.rs @@ -0,0 +1,50 @@ +use nu_engine::eval_expression; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct Let; + +impl Command for Let { + fn name(&self) -> &str { + "let" + } + + fn usage(&self) -> &str { + "Create a variable and give it a value." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("let") + .required("var_name", SyntaxShape::VarWithOptType, "variable name") + .required( + "initial_value", + SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), + "equals sign followed by value", + ) + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let var_id = call.positional[0] + .as_var() + .expect("internal error: missing variable"); + + let keyword_expr = call.positional[1] + .as_keyword() + .expect("internal error: missing keyword"); + + let rhs = eval_expression(context, keyword_expr)?; + + //println!("Adding: {:?} to {}", rhs, var_id); + + context.add_var(var_id, rhs); + Ok(Value::Nothing { + span: call.positional[0].span, + }) + } +} diff --git a/crates/nu-command/src/let_env.rs b/crates/nu-command/src/let_env.rs new file mode 100644 index 0000000000..39ed4800ed --- /dev/null +++ b/crates/nu-command/src/let_env.rs @@ -0,0 +1,51 @@ +use nu_engine::eval_expression; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct LetEnv; + +impl Command for LetEnv { + fn name(&self) -> &str { + "let-env" + } + + fn usage(&self) -> &str { + "Create an environment variable and give it a value." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("let-env") + .required("var_name", SyntaxShape::String, "variable name") + .required( + "initial_value", + SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::String)), + "equals sign followed by value", + ) + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + _input: Value, + ) -> Result { + let env_var = call.positional[0] + .as_string() + .expect("internal error: missing variable"); + + let keyword_expr = call.positional[1] + .as_keyword() + .expect("internal error: missing keyword"); + + let rhs = eval_expression(context, keyword_expr)?; + let rhs = rhs.as_string()?; + + //println!("Adding: {:?} to {}", rhs, var_id); + + context.add_env_var(env_var, rhs); + Ok(Value::Nothing { + span: call.positional[0].span, + }) + } +} diff --git a/crates/nu-command/src/lib.rs b/crates/nu-command/src/lib.rs index 31e1bb209f..ab4d11b5d1 100644 --- a/crates/nu-command/src/lib.rs +++ b/crates/nu-command/src/lib.rs @@ -1,7 +1,19 @@ -#[cfg(test)] -mod tests { - #[test] - fn it_works() { - assert_eq!(2 + 2, 4); - } -} +mod alias; +mod benchmark; +mod build_string; +mod def; +mod default_context; +mod for_; +mod if_; +mod let_; +mod let_env; + +pub use alias::Alias; +pub use benchmark::Benchmark; +pub use build_string::BuildString; +pub use def::Def; +pub use default_context::create_default_context; +pub use for_::For; +pub use if_::If; +pub use let_::Let; +pub use let_env::LetEnv; diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index f173c19f5c..662bb0f057 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -1,5 +1,3 @@ -use std::time::Instant; - use nu_protocol::ast::{Block, Call, Expr, Expression, Operator, Statement}; use nu_protocol::engine::EvaluationContext; use nu_protocol::{IntoRowStream, IntoValueStream, ShellError, Span, Value}; @@ -14,11 +12,11 @@ pub fn eval_operator(op: &Expression) -> Result { } } -fn eval_call(state: &EvaluationContext, call: &Call) -> Result { - let engine_state = state.engine_state.borrow(); +fn eval_call(context: &EvaluationContext, call: &Call) -> Result { + let engine_state = context.engine_state.borrow(); let decl = engine_state.get_decl(call.decl_id); if let Some(block_id) = decl.get_custom_command() { - let state = state.enter_scope(); + let state = context.enter_scope(); for (arg, param) in call.positional.iter().zip( decl.signature() .required_positional @@ -35,158 +33,21 @@ fn eval_call(state: &EvaluationContext, call: &Call) -> Result { - let engine_state = state.engine_state.borrow(); - if val { - let block = engine_state.get_block(then_block); - let state = state.enter_scope(); - eval_block(&state, block) - } else if let Some(else_case) = else_case { - if let Some(else_expr) = else_case.as_keyword() { - if let Some(block_id) = else_expr.as_block() { - let block = engine_state.get_block(block_id); - let state = state.enter_scope(); - eval_block(&state, block) - } else { - eval_expression(state, else_expr) - } - } else { - eval_expression(state, else_case) - } - } else { - Ok(Value::Nothing { span }) - } - } - _ => Err(ShellError::CantConvert("bool".into(), result.span())), - } - } else if decl.signature().name == "build-string" { - let mut output = vec![]; - - for expr in &call.positional { - let val = eval_expression(state, expr)?; - - output.push(val.into_string()); - } - Ok(Value::String { - val: output.join(""), - span: call.head, - }) - } else if decl.signature().name == "benchmark" { - let block = call.positional[0] - .as_block() - .expect("internal error: expected block"); - let engine_state = state.engine_state.borrow(); - let block = engine_state.get_block(block); - - let state = state.enter_scope(); - let start_time = Instant::now(); - eval_block(&state, block)?; - let end_time = Instant::now(); - println!("{} ms", (end_time - start_time).as_millis()); - Ok(Value::Nothing { - span: call.positional[0].span, - }) - } else if decl.signature().name == "for" { - let var_id = call.positional[0] - .as_var() - .expect("internal error: missing variable"); - - let keyword_expr = call.positional[1] - .as_keyword() - .expect("internal error: missing keyword"); - let end_val = eval_expression(state, keyword_expr)?; - - let block = call.positional[2] - .as_block() - .expect("internal error: expected block"); - let engine_state = state.engine_state.borrow(); - let block = engine_state.get_block(block); - - let state = state.enter_scope(); - - let mut x = Value::Int { - val: 0, - span: Span::unknown(), - }; - - loop { - if x == end_val { - break; - } else { - state.add_var(var_id, x.clone()); - eval_block(&state, block)?; - } - if let Value::Int { ref mut val, .. } = x { - *val += 1 - } - } - Ok(Value::Nothing { - span: call.positional[0].span, - }) - } else if decl.signature().name == "vars" { - state.engine_state.borrow().print_vars(); - Ok(Value::Nothing { span: call.head }) - } else if decl.signature().name == "decls" { - state.engine_state.borrow().print_decls(); - Ok(Value::Nothing { span: call.head }) - } else if decl.signature().name == "blocks" { - state.engine_state.borrow().print_blocks(); - Ok(Value::Nothing { span: call.head }) - } else if decl.signature().name == "stack" { - state.print_stack(); - Ok(Value::Nothing { span: call.head }) - } else if decl.signature().name == "def" || decl.signature().name == "alias" { - Ok(Value::Nothing { span: call.head }) } else { - Err(ShellError::Unsupported(call.head)) + decl.run( + context, + call, + Value::Nothing { + span: Span::unknown(), + }, + ) } } -pub fn eval_expression(state: &EvaluationContext, expr: &Expression) -> Result { +pub fn eval_expression( + context: &EvaluationContext, + expr: &Expression, +) -> Result { match &expr.expr { Expr::Bool(b) => Ok(Value::Bool { val: *b, @@ -200,17 +61,17 @@ pub fn eval_expression(state: &EvaluationContext, expr: &Expression) -> Result state + Expr::Var(var_id) => context .get_var(*var_id) .map_err(move |_| ShellError::VariableNotFound(expr.span)), - Expr::Call(call) => eval_call(state, call), + Expr::Call(call) => eval_call(context, call), Expr::ExternalCall(_, _) => Err(ShellError::Unsupported(expr.span)), Expr::Operator(_) => Ok(Value::Nothing { span: expr.span }), Expr::BinaryOp(lhs, op, rhs) => { let op_span = op.span; - let lhs = eval_expression(state, lhs)?; + let lhs = eval_expression(context, lhs)?; let op = eval_operator(op)?; - let rhs = eval_expression(state, rhs)?; + let rhs = eval_expression(context, rhs)?; match op { Operator::Plus => lhs.add(op_span, &rhs), @@ -228,10 +89,10 @@ pub fn eval_expression(state: &EvaluationContext, expr: &Expression) -> Result { - let engine_state = state.engine_state.borrow(); + let engine_state = context.engine_state.borrow(); let block = engine_state.get_block(*block_id); - let state = state.enter_scope(); + let state = context.enter_scope(); eval_block(&state, block) } Expr::Block(block_id) => Ok(Value::Block { @@ -241,7 +102,7 @@ pub fn eval_expression(state: &EvaluationContext, expr: &Expression) -> Result { let mut output = vec![]; for expr in x { - output.push(eval_expression(state, expr)?); + output.push(eval_expression(context, expr)?); } Ok(Value::List { val: output.into_value_stream(), @@ -251,14 +112,14 @@ pub fn eval_expression(state: &EvaluationContext, expr: &Expression) -> Result { let mut output_headers = vec![]; for expr in headers { - output_headers.push(eval_expression(state, expr)?.as_string()?); + output_headers.push(eval_expression(context, expr)?.as_string()?); } let mut output_rows = vec![]; for val in vals { let mut row = vec![]; for expr in val { - row.push(eval_expression(state, expr)?); + row.push(eval_expression(context, expr)?); } output_rows.push(row); } @@ -268,7 +129,7 @@ pub fn eval_expression(state: &EvaluationContext, expr: &Expression) -> Result eval_expression(state, expr), + Expr::Keyword(_, _, expr) => eval_expression(context, expr), Expr::String(s) => Ok(Value::String { val: s.clone(), span: expr.span, diff --git a/crates/nu-parser/src/errors.rs b/crates/nu-parser/src/errors.rs index 5f16a34711..bcd388c279 100644 --- a/crates/nu-parser/src/errors.rs +++ b/crates/nu-parser/src/errors.rs @@ -1,5 +1,4 @@ -use nu_protocol::{engine::StateWorkingSet, Span, Type}; -use std::ops::Range; +use nu_protocol::{Span, Type}; #[derive(Debug)] pub enum ParseError { diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index 12e6584df8..60a3dbb0e8 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -7,7 +7,7 @@ use crate::{ use nu_protocol::{ ast::{Block, Call, Expr, Expression, Operator, Pipeline, Statement}, engine::StateWorkingSet, - span, BlockId, DeclId, Flag, PositionalArg, Signature, Span, SyntaxShape, Type, VarId, + span, Flag, PositionalArg, Signature, Span, SyntaxShape, Type, VarId, }; #[derive(Debug, Clone)] @@ -416,9 +416,6 @@ pub fn parse_internal_call( // Parse a positional arg if there is one if let Some(positional) = signature.get_positional(positional_idx) { - //Make sure we leave enough spans for the remaining positionals - let decl = working_set.get_decl(decl_id); - let end = calculate_end_span(working_set, &signature, spans, spans_idx, positional_idx); // println!( diff --git a/crates/nu-protocol/src/engine/command.rs b/crates/nu-protocol/src/engine/command.rs index 8e8118fc8d..246985fe38 100644 --- a/crates/nu-protocol/src/engine/command.rs +++ b/crates/nu-protocol/src/engine/command.rs @@ -1,6 +1,6 @@ -use crate::{BlockId, Example, ShellError, Signature, Value}; +use crate::{ast::Call, BlockId, Example, ShellError, Signature, Value}; -use super::CommandArgs; +use super::EvaluationContext; pub trait Command { fn name(&self) -> &str; @@ -15,19 +15,12 @@ pub trait Command { "" } - fn run(&self, args: CommandArgs) -> Result; - - // fn run(&self, args: CommandArgs) -> Result { - // let context = args.context.clone(); - // let stream = self.run_with_actions(args)?; - - // Ok(Box::new(crate::evaluate::internal::InternalIterator { - // context, - // input: stream, - // leftovers: InputStream::empty(), - // }) - // .into_output_stream()) - // } + fn run( + &self, + context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result; fn is_binary(&self) -> bool { false diff --git a/crates/nu-protocol/src/engine/command_args.rs b/crates/nu-protocol/src/engine/command_args.rs deleted file mode 100644 index a9468e0118..0000000000 --- a/crates/nu-protocol/src/engine/command_args.rs +++ /dev/null @@ -1,7 +0,0 @@ -use super::{EvaluationContext, UnevaluatedCallInfo}; - -pub struct CommandArgs { - pub context: EvaluationContext, - pub call_info: UnevaluatedCallInfo, - pub input: crate::Value, -} diff --git a/crates/nu-protocol/src/engine/mod.rs b/crates/nu-protocol/src/engine/mod.rs index 68ffac9616..e6b7eb5d3d 100644 --- a/crates/nu-protocol/src/engine/mod.rs +++ b/crates/nu-protocol/src/engine/mod.rs @@ -1,11 +1,9 @@ mod call_info; mod command; -mod command_args; mod engine_state; mod evaluation_context; pub use call_info::*; pub use command::*; -pub use command_args::*; pub use engine_state::*; pub use evaluation_context::*; diff --git a/crates/nu-protocol/src/signature.rs b/crates/nu-protocol/src/signature.rs index a797ecbf4b..aa89f9474c 100644 --- a/crates/nu-protocol/src/signature.rs +++ b/crates/nu-protocol/src/signature.rs @@ -1,6 +1,9 @@ +use crate::ast::Call; use crate::engine::Command; +use crate::engine::EvaluationContext; use crate::BlockId; use crate::SyntaxShape; +use crate::Value; use crate::VarId; #[derive(Debug, Clone)] @@ -315,7 +318,12 @@ impl Command for Predeclaration { &self.signature.usage } - fn run(&self, _args: crate::engine::CommandArgs) -> Result { + fn run( + &self, + _context: &EvaluationContext, + _call: &Call, + _input: Value, + ) -> Result { panic!("Internal error: can't run a predeclaration without a body") } } @@ -338,7 +346,12 @@ impl Command for BlockCommand { &self.signature.usage } - fn run(&self, _args: crate::engine::CommandArgs) -> Result { + fn run( + &self, + _context: &EvaluationContext, + _call: &Call, + _input: Value, + ) -> Result { panic!("Internal error: can't run custom command with 'run', use block_id"); } diff --git a/src/main.rs b/src/main.rs index 030386a055..f1d595f89f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ -use nu_cli::{create_default_context, report_parsing_error, report_shell_error, NuHighlighter}; +use nu_cli::{report_parsing_error, report_shell_error, NuHighlighter}; +use nu_command::create_default_context; use nu_engine::eval_block; use nu_parser::parse_file; use nu_protocol::engine::{EngineState, EvaluationContext, StateWorkingSet}; @@ -68,8 +69,19 @@ fn main() -> std::io::Result<()> { Ok(Signal::Success(s)) => { if s.trim() == "exit" { break; + } else if s.trim() == "vars" { + engine_state.borrow().print_vars(); + continue; + } else if s.trim() == "decls" { + engine_state.borrow().print_decls(); + continue; + } else if s.trim() == "blocks" { + engine_state.borrow().print_blocks(); + continue; + } else if s.trim() == "stack" { + stack.print_stack(); + continue; } - // println!("input: '{}'", s); let (block, delta) = { let engine_state = engine_state.borrow(); From df63490266c4ce2a2a3a703a1f26910f006654f5 Mon Sep 17 00:00:00 2001 From: JT Date: Fri, 3 Sep 2021 14:15:01 +1200 Subject: [PATCH 06/13] Fix up calls and pipelines --- crates/nu-command/src/benchmark.rs | 2 +- crates/nu-command/src/default_context.rs | 4 +- crates/nu-command/src/for_.rs | 4 +- crates/nu-command/src/if_.rs | 6 +-- crates/nu-command/src/length.rs | 49 ++++++++++++++++++++++++ crates/nu-command/src/lib.rs | 2 + crates/nu-engine/src/eval.rs | 47 +++++++++++++---------- crates/nu-parser/src/flatten.rs | 1 - crates/nu-parser/src/parser.rs | 30 +++++++-------- crates/nu-parser/tests/test_parser.rs | 39 ++++++++++++------- crates/nu-protocol/src/ast/pipeline.rs | 4 ++ crates/nu-protocol/src/ast/statement.rs | 3 +- crates/nu-protocol/src/value.rs | 6 +++ src/main.rs | 9 +++-- 14 files changed, 145 insertions(+), 61 deletions(-) diff --git a/crates/nu-command/src/benchmark.rs b/crates/nu-command/src/benchmark.rs index f494da794e..5bd5a6afcc 100644 --- a/crates/nu-command/src/benchmark.rs +++ b/crates/nu-command/src/benchmark.rs @@ -34,7 +34,7 @@ impl Command for Benchmark { let state = context.enter_scope(); let start_time = Instant::now(); - eval_block(&state, block)?; + eval_block(&state, block, Value::nothing())?; let end_time = Instant::now(); println!("{} ms", (end_time - start_time).as_millis()); Ok(Value::Nothing { diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index f1bb536ff0..4b4f38c5dc 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -5,7 +5,7 @@ use nu_protocol::{ Signature, SyntaxShape, }; -use crate::{Alias, Benchmark, BuildString, Def, For, If, Let, LetEnv}; +use crate::{Alias, Benchmark, BuildString, Def, For, If, Length, Let, LetEnv}; pub fn create_default_context() -> Rc> { let engine_state = Rc::new(RefCell::new(EngineState::new())); @@ -33,6 +33,8 @@ pub fn create_default_context() -> Rc> { working_set.add_decl(Box::new(Benchmark)); + working_set.add_decl(Box::new(Length)); + let sig = Signature::build("exit"); working_set.add_decl(sig.predeclare()); let sig = Signature::build("vars"); diff --git a/crates/nu-command/src/for_.rs b/crates/nu-command/src/for_.rs index 8eaf393e72..3cede35e86 100644 --- a/crates/nu-command/src/for_.rs +++ b/crates/nu-command/src/for_.rs @@ -33,7 +33,7 @@ impl Command for For { &self, context: &EvaluationContext, call: &Call, - _input: Value, + input: Value, ) -> Result { let var_id = call.positional[0] .as_var() @@ -62,7 +62,7 @@ impl Command for For { break; } else { state.add_var(var_id, x.clone()); - eval_block(&state, block)?; + eval_block(&state, block, input.clone())?; } if let Value::Int { ref mut val, .. } = x { *val += 1 diff --git a/crates/nu-command/src/if_.rs b/crates/nu-command/src/if_.rs index 4124dbc7b8..bb785c4c2c 100644 --- a/crates/nu-command/src/if_.rs +++ b/crates/nu-command/src/if_.rs @@ -29,7 +29,7 @@ impl Command for If { &self, context: &EvaluationContext, call: &Call, - _input: Value, + input: Value, ) -> Result { let cond = &call.positional[0]; let then_block = call.positional[1] @@ -44,13 +44,13 @@ impl Command for If { if val { let block = engine_state.get_block(then_block); let state = context.enter_scope(); - eval_block(&state, block) + eval_block(&state, block, input) } else if let Some(else_case) = else_case { if let Some(else_expr) = else_case.as_keyword() { if let Some(block_id) = else_expr.as_block() { let block = engine_state.get_block(block_id); let state = context.enter_scope(); - eval_block(&state, block) + eval_block(&state, block, input) } else { eval_expression(context, else_expr) } diff --git a/crates/nu-command/src/length.rs b/crates/nu-command/src/length.rs index e69de29bb2..79a92c7782 100644 --- a/crates/nu-command/src/length.rs +++ b/crates/nu-command/src/length.rs @@ -0,0 +1,49 @@ +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, Value}; + +pub struct Length; + +impl Command for Length { + fn name(&self) -> &str { + "length" + } + + fn usage(&self) -> &str { + "Count the number of elements in the input." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("length") + } + + fn run( + &self, + _context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result { + match input { + Value::List { val, .. } => { + let length = val.count(); + + Ok(Value::Int { + val: length as i64, + span: call.head, + }) + } + Value::Table { val, .. } => { + let length = val.count(); + + Ok(Value::Int { + val: length as i64, + span: call.head, + }) + } + _ => Ok(Value::Int { + val: 1, + span: call.head, + }), + } + } +} diff --git a/crates/nu-command/src/lib.rs b/crates/nu-command/src/lib.rs index ab4d11b5d1..1d26da00fe 100644 --- a/crates/nu-command/src/lib.rs +++ b/crates/nu-command/src/lib.rs @@ -5,6 +5,7 @@ mod def; mod default_context; mod for_; mod if_; +mod length; mod let_; mod let_env; @@ -15,5 +16,6 @@ pub use def::Def; pub use default_context::create_default_context; pub use for_::For; pub use if_::If; +pub use length::Length; pub use let_::Let; pub use let_env::LetEnv; diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index 662bb0f057..0a97143c26 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -1,6 +1,6 @@ use nu_protocol::ast::{Block, Call, Expr, Expression, Operator, Statement}; use nu_protocol::engine::EvaluationContext; -use nu_protocol::{IntoRowStream, IntoValueStream, ShellError, Span, Value}; +use nu_protocol::{IntoRowStream, IntoValueStream, ShellError, Value}; pub fn eval_operator(op: &Expression) -> Result { match op { @@ -12,7 +12,7 @@ pub fn eval_operator(op: &Expression) -> Result { } } -fn eval_call(context: &EvaluationContext, call: &Call) -> Result { +fn eval_call(context: &EvaluationContext, call: &Call, input: Value) -> Result { let engine_state = context.engine_state.borrow(); let decl = engine_state.get_decl(call.decl_id); if let Some(block_id) = decl.get_custom_command() { @@ -32,15 +32,9 @@ fn eval_call(context: &EvaluationContext, call: &Call) -> Result context .get_var(*var_id) .map_err(move |_| ShellError::VariableNotFound(expr.span)), - Expr::Call(call) => eval_call(context, call), + Expr::Call(_) => panic!("Internal error: calls should be handled by eval_block"), Expr::ExternalCall(_, _) => Err(ShellError::Unsupported(expr.span)), Expr::Operator(_) => Ok(Value::Nothing { span: expr.span }), Expr::BinaryOp(lhs, op, rhs) => { @@ -93,7 +87,7 @@ pub fn eval_expression( let block = engine_state.get_block(*block_id); let state = context.enter_scope(); - eval_block(&state, block) + eval_block(&state, block, Value::nothing()) } Expr::Block(block_id) => Ok(Value::Block { val: *block_id, @@ -139,16 +133,29 @@ pub fn eval_expression( } } -pub fn eval_block(state: &EvaluationContext, block: &Block) -> Result { - let mut last = Ok(Value::Nothing { - span: Span { start: 0, end: 0 }, - }); - +pub fn eval_block( + context: &EvaluationContext, + block: &Block, + mut input: Value, +) -> Result { for stmt in &block.stmts { - if let Statement::Expression(expression) = stmt { - last = Ok(eval_expression(state, expression)?); + if let Statement::Pipeline(pipeline) = stmt { + for elem in &pipeline.expressions { + match elem { + Expression { + expr: Expr::Call(call), + .. + } => { + input = eval_call(context, call, input)?; + } + + elem => { + input = eval_expression(context, elem)?; + } + } + } } } - last + Ok(input) } diff --git a/crates/nu-parser/src/flatten.rs b/crates/nu-parser/src/flatten.rs index a2b490bf82..d2a55e9af6 100644 --- a/crates/nu-parser/src/flatten.rs +++ b/crates/nu-parser/src/flatten.rs @@ -29,7 +29,6 @@ pub fn flatten_statement( stmt: &Statement, ) -> Vec<(Span, FlatShape)> { match stmt { - Statement::Expression(expr) => flatten_expression(working_set, expr), Statement::Pipeline(pipeline) => flatten_pipeline(working_set, pipeline), _ => vec![], } diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index 60a3dbb0e8..0c1810c221 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -2066,30 +2066,30 @@ pub fn parse_def( }); ( - Statement::Expression(Expression { + Statement::Pipeline(Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), span: span(spans), ty: Type::Unknown, - }), + }])), error, ) } _ => ( - Statement::Expression(Expression { + Statement::Pipeline(Pipeline::from_vec(vec![Expression { expr: Expr::Garbage, span: span(spans), ty: Type::Unknown, - }), + }])), error, ), } } else { ( - Statement::Expression(Expression { + Statement::Pipeline(Pipeline::from_vec(vec![Expression { expr: Expr::Garbage, span: span(spans), ty: Type::Unknown, - }), + }])), Some(ParseError::UnknownState( "internal error: definition unparseable".into(), span(spans), @@ -2130,22 +2130,22 @@ pub fn parse_alias( } return ( - Statement::Expression(Expression { + Statement::Pipeline(Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), span: call_span, ty: Type::Unknown, - }), + }])), None, ); } } ( - Statement::Expression(Expression { + Statement::Pipeline(Pipeline::from_vec(vec![Expression { expr: Expr::Garbage, span: span(spans), ty: Type::Unknown, - }), + }])), Some(ParseError::UnknownState( "internal error: let statement unparseable".into(), span(spans), @@ -2175,21 +2175,21 @@ pub fn parse_let( } return ( - Statement::Expression(Expression { + Statement::Pipeline(Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), span: call_span, ty: Type::Unknown, - }), + }])), err, ); } } ( - Statement::Expression(Expression { + Statement::Pipeline(Pipeline::from_vec(vec![Expression { expr: Expr::Garbage, span: span(spans), ty: Type::Unknown, - }), + }])), Some(ParseError::UnknownState( "internal error: let statement unparseable".into(), span(spans), @@ -2210,7 +2210,7 @@ pub fn parse_statement( (stmt, None) } else { let (expr, err) = parse_expression(working_set, spans); - (Statement::Expression(expr), err) + (Statement::Pipeline(Pipeline::from_vec(vec![expr])), err) } } diff --git a/crates/nu-parser/tests/test_parser.rs b/crates/nu-parser/tests/test_parser.rs index 959925d96a..0ffa726de1 100644 --- a/crates/nu-parser/tests/test_parser.rs +++ b/crates/nu-parser/tests/test_parser.rs @@ -1,7 +1,7 @@ use nu_parser::ParseError; use nu_parser::*; use nu_protocol::{ - ast::{Expr, Expression, Statement}, + ast::{Expr, Expression, Pipeline, Statement}, engine::{EngineState, StateWorkingSet}, Signature, SyntaxShape, }; @@ -15,13 +15,21 @@ pub fn parse_int() { assert!(err.is_none()); assert!(block.len() == 1); - assert!(matches!( - block[0], - Statement::Expression(Expression { - expr: Expr::Int(3), - .. - }) - )); + match &block[0] { + Statement::Pipeline(Pipeline { + expressions: expressions, + }) => { + assert!(expressions.len() == 1); + assert!(matches!( + expressions[0], + Expression { + expr: Expr::Int(3), + .. + } + )) + } + _ => panic!("No match"), + } } #[test] @@ -38,11 +46,16 @@ pub fn parse_call() { assert!(block.len() == 1); match &block[0] { - Statement::Expression(Expression { - expr: Expr::Call(call), - .. - }) => { - assert_eq!(call.decl_id, 0); + Statement::Pipeline(Pipeline { expressions }) => { + assert_eq!(expressions.len(), 1); + + if let Expression { + expr: Expr::Call(call), + .. + } = &expressions[0] + { + assert_eq!(call.decl_id, 0); + } } _ => panic!("not a call"), } diff --git a/crates/nu-protocol/src/ast/pipeline.rs b/crates/nu-protocol/src/ast/pipeline.rs index fd4425290c..1f5652fa41 100644 --- a/crates/nu-protocol/src/ast/pipeline.rs +++ b/crates/nu-protocol/src/ast/pipeline.rs @@ -17,4 +17,8 @@ impl Pipeline { expressions: vec![], } } + + pub fn from_vec(expressions: Vec) -> Pipeline { + Self { expressions } + } } diff --git a/crates/nu-protocol/src/ast/statement.rs b/crates/nu-protocol/src/ast/statement.rs index 24703a8cfb..5fa7faef12 100644 --- a/crates/nu-protocol/src/ast/statement.rs +++ b/crates/nu-protocol/src/ast/statement.rs @@ -1,9 +1,8 @@ -use super::{Expression, Pipeline}; +use super::Pipeline; use crate::DeclId; #[derive(Debug, Clone)] pub enum Statement { Declaration(DeclId), Pipeline(Pipeline), - Expression(Expression), } diff --git a/crates/nu-protocol/src/value.rs b/crates/nu-protocol/src/value.rs index bd328eb714..3f3ce34e2b 100644 --- a/crates/nu-protocol/src/value.rs +++ b/crates/nu-protocol/src/value.rs @@ -197,6 +197,12 @@ impl Value { Value::Nothing { .. } => String::new(), } } + + pub fn nothing() -> Value { + Value::Nothing { + span: Span::unknown(), + } + } } impl PartialEq for Value { diff --git a/src/main.rs b/src/main.rs index f1d595f89f..0c649d2748 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,7 +2,10 @@ use nu_cli::{report_parsing_error, report_shell_error, NuHighlighter}; use nu_command::create_default_context; use nu_engine::eval_block; use nu_parser::parse_file; -use nu_protocol::engine::{EngineState, EvaluationContext, StateWorkingSet}; +use nu_protocol::{ + engine::{EngineState, EvaluationContext, StateWorkingSet}, + Value, +}; #[cfg(test)] mod tests; @@ -32,7 +35,7 @@ fn main() -> std::io::Result<()> { stack: nu_protocol::engine::Stack::new(), }; - match eval_block(&state, &block) { + match eval_block(&state, &block, Value::nothing()) { Ok(value) => { println!("{}", value.into_string()); } @@ -106,7 +109,7 @@ fn main() -> std::io::Result<()> { stack: stack.clone(), }; - match eval_block(&state, &block) { + match eval_block(&state, &block, Value::nothing()) { Ok(value) => { println!("{}", value.into_string()); } From 750502c8701200ed7cee8977c8ed0cefbefd3e81 Mon Sep 17 00:00:00 2001 From: JT Date: Fri, 3 Sep 2021 14:57:18 +1200 Subject: [PATCH 07/13] Fix up for_in --- crates/nu-command/src/for_.rs | 45 +++++++++---------- crates/nu-command/src/length.rs | 4 ++ crates/nu-engine/src/eval.rs | 2 +- .../src/engine/evaluation_context.rs | 1 + crates/nu-protocol/src/value.rs | 13 ++++-- 5 files changed, 38 insertions(+), 27 deletions(-) diff --git a/crates/nu-command/src/for_.rs b/crates/nu-command/src/for_.rs index 3cede35e86..b240d30cab 100644 --- a/crates/nu-command/src/for_.rs +++ b/crates/nu-command/src/for_.rs @@ -1,7 +1,7 @@ use nu_engine::{eval_block, eval_expression}; use nu_protocol::ast::Call; use nu_protocol::engine::{Command, EvaluationContext}; -use nu_protocol::{Signature, Span, SyntaxShape, Value}; +use nu_protocol::{IntoValueStream, Signature, Span, SyntaxShape, Value}; pub struct For; @@ -23,7 +23,10 @@ impl Command for For { ) .required( "range", - SyntaxShape::Keyword(b"in".to_vec(), Box::new(SyntaxShape::Int)), + SyntaxShape::Keyword( + b"in".to_vec(), + Box::new(SyntaxShape::List(Box::new(SyntaxShape::Int))), + ), "range of the loop", ) .required("block", SyntaxShape::Block, "the block to run") @@ -42,34 +45,30 @@ impl Command for For { let keyword_expr = call.positional[1] .as_keyword() .expect("internal error: missing keyword"); - let end_val = eval_expression(context, keyword_expr)?; + let values = eval_expression(context, keyword_expr)?; let block = call.positional[2] .as_block() .expect("internal error: expected block"); - let engine_state = context.engine_state.borrow(); - let block = engine_state.get_block(block); + let context = context.clone(); - let state = context.enter_scope(); + match values { + Value::List { val, .. } => Ok(Value::List { + val: val + .map(move |x| { + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block); - let mut x = Value::Int { - val: 0, - span: Span::unknown(), - }; + let state = context.enter_scope(); + state.add_var(var_id, x.clone()); - loop { - if x == end_val { - break; - } else { - state.add_var(var_id, x.clone()); - eval_block(&state, block, input.clone())?; - } - if let Value::Int { ref mut val, .. } = x { - *val += 1 - } + //FIXME: DON'T UNWRAP + eval_block(&state, block, input.clone()).unwrap() + }) + .into_value_stream(), + span: call.head, + }), + _ => Ok(Value::nothing()), } - Ok(Value::Nothing { - span: call.positional[0].span, - }) } } diff --git a/crates/nu-command/src/length.rs b/crates/nu-command/src/length.rs index 79a92c7782..2706948943 100644 --- a/crates/nu-command/src/length.rs +++ b/crates/nu-command/src/length.rs @@ -40,6 +40,10 @@ impl Command for Length { span: call.head, }) } + Value::Nothing { .. } => Ok(Value::Int { + val: 0, + span: call.head, + }), _ => Ok(Value::Int { val: 1, span: call.head, diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index 0a97143c26..e8e0f4c04e 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -99,7 +99,7 @@ pub fn eval_expression( output.push(eval_expression(context, expr)?); } Ok(Value::List { - val: output.into_value_stream(), + val: output.into_iter().into_value_stream(), span: expr.span, }) } diff --git a/crates/nu-protocol/src/engine/evaluation_context.rs b/crates/nu-protocol/src/engine/evaluation_context.rs index f1baa94949..5229659343 100644 --- a/crates/nu-protocol/src/engine/evaluation_context.rs +++ b/crates/nu-protocol/src/engine/evaluation_context.rs @@ -3,6 +3,7 @@ use std::{cell::RefCell, collections::HashMap, rc::Rc}; use crate::{ShellError, Value, VarId}; +#[derive(Clone)] pub struct EvaluationContext { pub engine_state: Rc>, pub stack: Stack, diff --git a/crates/nu-protocol/src/value.rs b/crates/nu-protocol/src/value.rs index 3f3ce34e2b..640fbe7d41 100644 --- a/crates/nu-protocol/src/value.rs +++ b/crates/nu-protocol/src/value.rs @@ -5,7 +5,7 @@ use crate::{span, BlockId, Span, Type}; use crate::ShellError; #[derive(Clone)] -pub struct ValueStream(Rc>>); +pub struct ValueStream(pub Rc>>); impl ValueStream { pub fn into_string(self) -> String { @@ -18,6 +18,10 @@ impl ValueStream { .join(", ".into()) ) } + + pub fn from_stream(input: impl Iterator + 'static) -> ValueStream { + ValueStream(Rc::new(RefCell::new(input))) + } } impl Debug for ValueStream { @@ -41,9 +45,12 @@ pub trait IntoValueStream { fn into_value_stream(self) -> ValueStream; } -impl IntoValueStream for Vec { +impl IntoValueStream for T +where + T: Iterator + 'static, +{ fn into_value_stream(self) -> ValueStream { - ValueStream(Rc::new(RefCell::new(self.into_iter()))) + ValueStream::from_stream(self) } } From 6c0ce95d0f9fdec8d6f71e7ea2e2c2218531c088 Mon Sep 17 00:00:00 2001 From: JT Date: Fri, 3 Sep 2021 15:45:34 +1200 Subject: [PATCH 08/13] Add simple each --- crates/nu-command/src/default_context.rs | 4 +- crates/nu-command/src/each.rs | 61 ++++++++++++++++++++++++ crates/nu-command/src/for_.rs | 2 +- crates/nu-command/src/lib.rs | 2 + 4 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 crates/nu-command/src/each.rs diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index 4b4f38c5dc..e4f5191827 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -5,7 +5,7 @@ use nu_protocol::{ Signature, SyntaxShape, }; -use crate::{Alias, Benchmark, BuildString, Def, For, If, Length, Let, LetEnv}; +use crate::{Alias, Benchmark, BuildString, Def, Each, For, If, Length, Let, LetEnv}; pub fn create_default_context() -> Rc> { let engine_state = Rc::new(RefCell::new(EngineState::new())); @@ -31,6 +31,8 @@ pub fn create_default_context() -> Rc> { working_set.add_decl(Box::new(For)); + working_set.add_decl(Box::new(Each)); + working_set.add_decl(Box::new(Benchmark)); working_set.add_decl(Box::new(Length)); diff --git a/crates/nu-command/src/each.rs b/crates/nu-command/src/each.rs new file mode 100644 index 0000000000..6a96ec9e74 --- /dev/null +++ b/crates/nu-command/src/each.rs @@ -0,0 +1,61 @@ +use nu_engine::{eval_block, eval_expression}; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{IntoValueStream, Signature, Span, SyntaxShape, Value}; + +pub struct Each; + +impl Command for Each { + fn name(&self) -> &str { + "each" + } + + fn usage(&self) -> &str { + "Run a block on each element of input" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("each") + .required( + "var_name", + SyntaxShape::Variable, + "name of the looping variable", + ) + .required("block", SyntaxShape::Block, "the block to run") + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result { + let var_id = call.positional[0] + .as_var() + .expect("internal error: missing variable"); + + let block = call.positional[1] + .as_block() + .expect("internal error: expected block"); + let context = context.clone(); + + match input { + Value::List { val, .. } => Ok(Value::List { + val: val + .map(move |x| { + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block); + + let state = context.enter_scope(); + state.add_var(var_id, x.clone()); + + //FIXME: DON'T UNWRAP + eval_block(&state, block, Value::nothing()).unwrap() + }) + .into_value_stream(), + span: call.head, + }), + _ => Ok(Value::nothing()), + } + } +} diff --git a/crates/nu-command/src/for_.rs b/crates/nu-command/src/for_.rs index b240d30cab..bdf882ac0a 100644 --- a/crates/nu-command/src/for_.rs +++ b/crates/nu-command/src/for_.rs @@ -63,7 +63,7 @@ impl Command for For { state.add_var(var_id, x.clone()); //FIXME: DON'T UNWRAP - eval_block(&state, block, input.clone()).unwrap() + eval_block(&state, block, Value::nothing()).unwrap() }) .into_value_stream(), span: call.head, diff --git a/crates/nu-command/src/lib.rs b/crates/nu-command/src/lib.rs index 1d26da00fe..1df25ef519 100644 --- a/crates/nu-command/src/lib.rs +++ b/crates/nu-command/src/lib.rs @@ -3,6 +3,7 @@ mod benchmark; mod build_string; mod def; mod default_context; +mod each; mod for_; mod if_; mod length; @@ -14,6 +15,7 @@ pub use benchmark::Benchmark; pub use build_string::BuildString; pub use def::Def; pub use default_context::create_default_context; +pub use each::Each; pub use for_::For; pub use if_::If; pub use length::Length; From 12d80c2732fd6ff2e86c90e75ca4e29227c0c76c Mon Sep 17 00:00:00 2001 From: JT Date: Fri, 3 Sep 2021 15:49:14 +1200 Subject: [PATCH 09/13] Fix test --- crates/nu-engine/src/eval.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index e8e0f4c04e..0ec959c6ae 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -58,7 +58,7 @@ pub fn eval_expression( Expr::Var(var_id) => context .get_var(*var_id) .map_err(move |_| ShellError::VariableNotFound(expr.span)), - Expr::Call(_) => panic!("Internal error: calls should be handled by eval_block"), + Expr::Call(call) => eval_call(context, call, Value::nothing()), Expr::ExternalCall(_, _) => Err(ShellError::Unsupported(expr.span)), Expr::Operator(_) => Ok(Value::Nothing { span: expr.span }), Expr::BinaryOp(lhs, op, rhs) => { From 82cf6caba4873004112c21fe2d2b111ee9e67b01 Mon Sep 17 00:00:00 2001 From: JT Date: Fri, 3 Sep 2021 16:01:45 +1200 Subject: [PATCH 10/13] Add do --- crates/nu-command/src/default_context.rs | 4 ++- crates/nu-command/src/do_.rs | 40 ++++++++++++++++++++++++ crates/nu-command/src/lib.rs | 2 ++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 crates/nu-command/src/do_.rs diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index e4f5191827..fbaaaf537b 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -5,7 +5,7 @@ use nu_protocol::{ Signature, SyntaxShape, }; -use crate::{Alias, Benchmark, BuildString, Def, Each, For, If, Length, Let, LetEnv}; +use crate::{Alias, Benchmark, BuildString, Def, Do, Each, For, If, Length, Let, LetEnv}; pub fn create_default_context() -> Rc> { let engine_state = Rc::new(RefCell::new(EngineState::new())); @@ -33,6 +33,8 @@ pub fn create_default_context() -> Rc> { working_set.add_decl(Box::new(Each)); + working_set.add_decl(Box::new(Do)); + working_set.add_decl(Box::new(Benchmark)); working_set.add_decl(Box::new(Length)); diff --git a/crates/nu-command/src/do_.rs b/crates/nu-command/src/do_.rs new file mode 100644 index 0000000000..dba61b6024 --- /dev/null +++ b/crates/nu-command/src/do_.rs @@ -0,0 +1,40 @@ +use nu_engine::{eval_block, eval_expression}; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EvaluationContext}; +use nu_protocol::{Signature, SyntaxShape, Value}; + +pub struct Do; + +impl Command for Do { + fn name(&self) -> &str { + "do" + } + + fn usage(&self) -> &str { + "Run a block" + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("do").required("block", SyntaxShape::Block, "the block to run") + } + + fn run( + &self, + context: &EvaluationContext, + call: &Call, + input: Value, + ) -> Result { + let block = &call.positional[0]; + + let out = eval_expression(context, &block)?; + + match out { + Value::Block { val: block_id, .. } => { + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block_id); + eval_block(context, block, input) + } + _ => Ok(Value::nothing()), + } + } +} diff --git a/crates/nu-command/src/lib.rs b/crates/nu-command/src/lib.rs index 1df25ef519..3e7d99e7d7 100644 --- a/crates/nu-command/src/lib.rs +++ b/crates/nu-command/src/lib.rs @@ -3,6 +3,7 @@ mod benchmark; mod build_string; mod def; mod default_context; +mod do_; mod each; mod for_; mod if_; @@ -15,6 +16,7 @@ pub use benchmark::Benchmark; pub use build_string::BuildString; pub use def::Def; pub use default_context::create_default_context; +pub use do_::Do; pub use each::Each; pub use for_::For; pub use if_::If; From b20c4047d48d38792d99e699c2e2556f57e96fc6 Mon Sep 17 00:00:00 2001 From: JT Date: Fri, 3 Sep 2021 19:35:29 +1200 Subject: [PATCH 11/13] Some cleanup, better subexpressions --- crates/nu-command/src/each.rs | 4 ++-- crates/nu-command/src/for_.rs | 4 ++-- crates/nu-parser/src/parser.rs | 28 ++++++++------------------- crates/nu-parser/tests/test_parser.rs | 4 +--- 4 files changed, 13 insertions(+), 27 deletions(-) diff --git a/crates/nu-command/src/each.rs b/crates/nu-command/src/each.rs index 6a96ec9e74..2ec432f0a6 100644 --- a/crates/nu-command/src/each.rs +++ b/crates/nu-command/src/each.rs @@ -1,7 +1,7 @@ -use nu_engine::{eval_block, eval_expression}; +use nu_engine::eval_block; use nu_protocol::ast::Call; use nu_protocol::engine::{Command, EvaluationContext}; -use nu_protocol::{IntoValueStream, Signature, Span, SyntaxShape, Value}; +use nu_protocol::{IntoValueStream, Signature, SyntaxShape, Value}; pub struct Each; diff --git a/crates/nu-command/src/for_.rs b/crates/nu-command/src/for_.rs index bdf882ac0a..c72d5b2147 100644 --- a/crates/nu-command/src/for_.rs +++ b/crates/nu-command/src/for_.rs @@ -1,7 +1,7 @@ use nu_engine::{eval_block, eval_expression}; use nu_protocol::ast::Call; use nu_protocol::engine::{Command, EvaluationContext}; -use nu_protocol::{IntoValueStream, Signature, Span, SyntaxShape, Value}; +use nu_protocol::{IntoValueStream, Signature, SyntaxShape, Value}; pub struct For; @@ -36,7 +36,7 @@ impl Command for For { &self, context: &EvaluationContext, call: &Call, - input: Value, + _input: Value, ) -> Result { let var_id = call.positional[0] .as_var() diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index 0c1810c221..092335922a 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -595,11 +595,7 @@ pub fn parse_call( } } -pub fn parse_int( - working_set: &mut StateWorkingSet, - token: &str, - span: Span, -) -> (Expression, Option) { +pub fn parse_int(token: &str, span: Span) -> (Expression, Option) { if let Some(token) = token.strip_prefix("0x") { if let Ok(v) = i64::from_str_radix(token, 16) { ( @@ -677,11 +673,7 @@ pub fn parse_int( } } -pub fn parse_float( - working_set: &mut StateWorkingSet, - token: &str, - span: Span, -) -> (Expression, Option) { +pub fn parse_float(token: &str, span: Span) -> (Expression, Option) { if let Ok(x) = token.parse::() { ( Expression { @@ -699,14 +691,10 @@ pub fn parse_float( } } -pub fn parse_number( - working_set: &mut StateWorkingSet, - token: &str, - span: Span, -) -> (Expression, Option) { - if let (x, None) = parse_int(working_set, token, span) { +pub fn parse_number(token: &str, span: Span) -> (Expression, Option) { + if let (x, None) = parse_int(token, span) { (x, None) - } else if let (x, None) = parse_float(working_set, token, span) { + } else if let (x, None) = parse_float(token, span) { (x, None) } else { ( @@ -951,7 +939,7 @@ pub fn parse_full_column_path( let source = working_set.get_span_contents(span); - let (output, err) = lex(source, start, &[], &[]); + let (output, err) = lex(source, start, &[b'\n'], &[]); error = error.or(err); let (output, err) = lite_parse(&output); @@ -1717,7 +1705,7 @@ pub fn parse_value( match shape { SyntaxShape::Number => { if let Ok(token) = String::from_utf8(bytes.into()) { - parse_number(working_set, &token, span) + parse_number(&token, span) } else { ( garbage(span), @@ -1727,7 +1715,7 @@ pub fn parse_value( } SyntaxShape::Int => { if let Ok(token) = String::from_utf8(bytes.into()) { - parse_int(working_set, &token, span) + parse_int(&token, span) } else { ( garbage(span), diff --git a/crates/nu-parser/tests/test_parser.rs b/crates/nu-parser/tests/test_parser.rs index 0ffa726de1..a2f5fc2609 100644 --- a/crates/nu-parser/tests/test_parser.rs +++ b/crates/nu-parser/tests/test_parser.rs @@ -16,9 +16,7 @@ pub fn parse_int() { assert!(err.is_none()); assert!(block.len() == 1); match &block[0] { - Statement::Pipeline(Pipeline { - expressions: expressions, - }) => { + Statement::Pipeline(Pipeline { expressions }) => { assert!(expressions.len() == 1); assert!(matches!( expressions[0], From 74bb2af3e199012cf9064f0a50c5c31c6b3fd850 Mon Sep 17 00:00:00 2001 From: JT Date: Sat, 4 Sep 2021 08:58:44 +1200 Subject: [PATCH 12/13] Fix up block parse recovery --- crates/nu-parser/src/parser.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index 092335922a..9529bb1419 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -1687,6 +1687,8 @@ pub fn parse_value( return parse_dollar_expr(working_set, span); } else if bytes.starts_with(b"(") { return parse_full_column_path(working_set, span); + } else if bytes.starts_with(b"{") { + return parse_block_expression(working_set, span); } else if bytes.starts_with(b"[") { match shape { SyntaxShape::Any From 5e33b8536b8cc0287b978b86445d73f5e6acdb78 Mon Sep 17 00:00:00 2001 From: JT Date: Sat, 4 Sep 2021 18:52:28 +1200 Subject: [PATCH 13/13] Add discrete list/table --- crates/nu-command/src/each.rs | 16 +++++++ crates/nu-command/src/for_.rs | 20 ++++++++- crates/nu-command/src/length.rs | 20 ++++++++- crates/nu-engine/src/eval.rs | 6 +-- crates/nu-parser/src/parser.rs | 9 +++- crates/nu-protocol/src/ty.rs | 4 ++ crates/nu-protocol/src/value.rs | 79 +++++++++++++++++++++++++++++++-- 7 files changed, 142 insertions(+), 12 deletions(-) diff --git a/crates/nu-command/src/each.rs b/crates/nu-command/src/each.rs index 2ec432f0a6..f2964b3798 100644 --- a/crates/nu-command/src/each.rs +++ b/crates/nu-command/src/each.rs @@ -42,6 +42,22 @@ impl Command for Each { match input { Value::List { val, .. } => Ok(Value::List { val: val + .into_iter() + .map(move |x| { + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block); + + let state = context.enter_scope(); + state.add_var(var_id, x.clone()); + + //FIXME: DON'T UNWRAP + eval_block(&state, block, Value::nothing()).unwrap() + }) + .collect(), + span: call.head, + }), + Value::ValueStream { stream, .. } => Ok(Value::ValueStream { + stream: stream .map(move |x| { let engine_state = context.engine_state.borrow(); let block = engine_state.get_block(block); diff --git a/crates/nu-command/src/for_.rs b/crates/nu-command/src/for_.rs index c72d5b2147..792a656011 100644 --- a/crates/nu-command/src/for_.rs +++ b/crates/nu-command/src/for_.rs @@ -53,8 +53,8 @@ impl Command for For { let context = context.clone(); match values { - Value::List { val, .. } => Ok(Value::List { - val: val + Value::ValueStream { stream, .. } => Ok(Value::ValueStream { + stream: stream .map(move |x| { let engine_state = context.engine_state.borrow(); let block = engine_state.get_block(block); @@ -68,6 +68,22 @@ impl Command for For { .into_value_stream(), span: call.head, }), + Value::List { val, .. } => Ok(Value::List { + val: val + .into_iter() + .map(move |x| { + let engine_state = context.engine_state.borrow(); + let block = engine_state.get_block(block); + + let state = context.enter_scope(); + state.add_var(var_id, x.clone()); + + //FIXME: DON'T UNWRAP + eval_block(&state, block, Value::nothing()).unwrap() + }) + .collect(), + span: call.head, + }), _ => Ok(Value::nothing()), } } diff --git a/crates/nu-command/src/length.rs b/crates/nu-command/src/length.rs index 2706948943..00247287ce 100644 --- a/crates/nu-command/src/length.rs +++ b/crates/nu-command/src/length.rs @@ -25,7 +25,7 @@ impl Command for Length { ) -> Result { match input { Value::List { val, .. } => { - let length = val.count(); + let length = val.len(); Ok(Value::Int { val: length as i64, @@ -33,7 +33,23 @@ impl Command for Length { }) } Value::Table { val, .. } => { - let length = val.count(); + let length = val.len(); + + Ok(Value::Int { + val: length as i64, + span: call.head, + }) + } + Value::ValueStream { stream, .. } => { + let length = stream.count(); + + Ok(Value::Int { + val: length as i64, + span: call.head, + }) + } + Value::RowStream { stream, .. } => { + let length = stream.count(); Ok(Value::Int { val: length as i64, diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index 0ec959c6ae..733fd63807 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -1,6 +1,6 @@ use nu_protocol::ast::{Block, Call, Expr, Expression, Operator, Statement}; use nu_protocol::engine::EvaluationContext; -use nu_protocol::{IntoRowStream, IntoValueStream, ShellError, Value}; +use nu_protocol::{ShellError, Value}; pub fn eval_operator(op: &Expression) -> Result { match op { @@ -99,7 +99,7 @@ pub fn eval_expression( output.push(eval_expression(context, expr)?); } Ok(Value::List { - val: output.into_iter().into_value_stream(), + val: output, span: expr.span, }) } @@ -119,7 +119,7 @@ pub fn eval_expression( } Ok(Value::Table { headers: output_headers, - val: output_rows.into_row_stream(), + val: output_rows, span: expr.span, }) } diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index 9529bb1419..8b9aefe656 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -1688,7 +1688,14 @@ pub fn parse_value( } else if bytes.starts_with(b"(") { return parse_full_column_path(working_set, span); } else if bytes.starts_with(b"{") { - return parse_block_expression(working_set, span); + if matches!(shape, SyntaxShape::Block) || matches!(shape, SyntaxShape::Any) { + return parse_block_expression(working_set, span); + } else { + return ( + Expression::garbage(span), + Some(ParseError::Expected("non-block value".into(), span)), + ); + } } else if bytes.starts_with(b"[") { match shape { SyntaxShape::Any diff --git a/crates/nu-protocol/src/ty.rs b/crates/nu-protocol/src/ty.rs index 80cbc3b5e6..ba2fe8d2ec 100644 --- a/crates/nu-protocol/src/ty.rs +++ b/crates/nu-protocol/src/ty.rs @@ -15,6 +15,8 @@ pub enum Type { Number, Nothing, Table, + RowStream, + ValueStream, Unknown, } @@ -34,6 +36,8 @@ impl Display for Type { Type::Number => write!(f, "number"), Type::String => write!(f, "string"), Type::Table => write!(f, "table"), + Type::ValueStream => write!(f, "value stream"), + Type::RowStream => write!(f, "row stream"), Type::Unknown => write!(f, "unknown"), } } diff --git a/crates/nu-protocol/src/value.rs b/crates/nu-protocol/src/value.rs index 640fbe7d41..c88bc4be60 100644 --- a/crates/nu-protocol/src/value.rs +++ b/crates/nu-protocol/src/value.rs @@ -125,13 +125,22 @@ pub enum Value { val: String, span: Span, }, + ValueStream { + stream: ValueStream, + span: Span, + }, + RowStream { + headers: Vec, + stream: RowStream, + span: Span, + }, List { - val: ValueStream, + val: Vec, span: Span, }, Table { headers: Vec, - val: RowStream, + val: Vec>, span: Span, }, Block { @@ -160,6 +169,8 @@ impl Value { Value::List { span, .. } => *span, Value::Table { span, .. } => *span, Value::Block { span, .. } => *span, + Value::RowStream { span, .. } => *span, + Value::ValueStream { span, .. } => *span, Value::Nothing { span, .. } => *span, } } @@ -170,6 +181,8 @@ impl Value { Value::Int { span, .. } => *span = new_span, Value::Float { span, .. } => *span = new_span, Value::String { span, .. } => *span = new_span, + Value::RowStream { span, .. } => *span = new_span, + Value::ValueStream { span, .. } => *span = new_span, Value::List { span, .. } => *span = new_span, Value::Table { span, .. } => *span = new_span, Value::Block { span, .. } => *span = new_span, @@ -189,6 +202,8 @@ impl Value { Value::Table { .. } => Type::Table, // FIXME Value::Nothing { .. } => Type::Nothing, Value::Block { .. } => Type::Block, + Value::ValueStream { .. } => Type::ValueStream, + Value::RowStream { .. } => Type::RowStream, } } @@ -198,8 +213,25 @@ impl Value { Value::Int { val, .. } => val.to_string(), Value::Float { val, .. } => val.to_string(), Value::String { val, .. } => val, - Value::List { val, .. } => val.into_string(), - Value::Table { headers, val, .. } => val.into_string(headers), + Value::ValueStream { stream, .. } => stream.into_string(), + Value::List { val, .. } => val + .into_iter() + .map(|x| x.into_string()) + .collect::>() + .join(", "), + Value::Table { val, .. } => val + .into_iter() + .map(|x| { + x.into_iter() + .map(|x| x.into_string()) + .collect::>() + .join(", ") + }) + .collect::>() + .join("\n"), + Value::RowStream { + headers, stream, .. + } => stream.into_string(headers), Value::Block { val, .. } => format!("", val), Value::Nothing { .. } => String::new(), } @@ -517,6 +549,25 @@ impl Value { val: lhs == rhs, span, }), + (Value::List { val: lhs, .. }, Value::List { val: rhs, .. }) => Ok(Value::Bool { + val: lhs == rhs, + span, + }), + ( + Value::Table { + val: lhs, + headers: lhs_headers, + .. + }, + Value::Table { + val: rhs, + headers: rhs_headers, + .. + }, + ) => Ok(Value::Bool { + val: lhs_headers == rhs_headers && lhs == rhs, + span, + }), _ => Err(ShellError::OperatorMismatch { op_span: op, lhs_ty: self.get_type(), @@ -553,6 +604,26 @@ impl Value { val: lhs != rhs, span, }), + (Value::List { val: lhs, .. }, Value::List { val: rhs, .. }) => Ok(Value::Bool { + val: lhs != rhs, + span, + }), + ( + Value::Table { + val: lhs, + headers: lhs_headers, + .. + }, + Value::Table { + val: rhs, + headers: rhs_headers, + .. + }, + ) => Ok(Value::Bool { + val: lhs_headers != rhs_headers || lhs != rhs, + span, + }), + _ => Err(ShellError::OperatorMismatch { op_span: op, lhs_ty: self.get_type(),