From a4bcc1ff3dcff5a95ddc318a6578cff32c86ab1d Mon Sep 17 00:00:00 2001 From: JT Date: Sat, 24 Jul 2021 17:57:17 +1200 Subject: [PATCH 1/2] WIP --- src/eval.rs | 13 ++- src/flatten.rs | 9 +- src/main.rs | 27 ++--- src/parser.rs | 244 ++++++++++++++++++++++++---------------- src/signature.rs | 61 +++++++++- src/syntax_highlight.rs | 3 + 6 files changed, 242 insertions(+), 115 deletions(-) diff --git a/src/eval.rs b/src/eval.rs index 13888be44d..30bf276fc6 100644 --- a/src/eval.rs +++ b/src/eval.rs @@ -13,6 +13,7 @@ pub enum ShellError { #[derive(Debug, Clone)] pub enum Value { + Bool { val: bool, span: Span }, Int { val: i64, span: Span }, String { val: String, span: Span }, List(Vec), @@ -92,7 +93,11 @@ fn eval_call(state: &State, stack: &mut Stack, call: &Call) -> Result Result { match &expr.expr { + Expr::Bool(b) => Ok(Value::Bool { + val: *b, + span: expr.span, + }), Expr::Int(i) => Ok(Value::Int { val: *i, span: expr.span, @@ -143,7 +152,7 @@ pub fn eval_expression( Ok(Value::List(output)) } Expr::Table(_, _) => Err(ShellError::Unsupported(expr.span)), - Expr::Literal(_) => Ok(Value::Unknown), + Expr::Keyword(_, expr) => eval_expression(state, stack, expr), Expr::String(s) => Ok(Value::String { val: s.clone(), span: expr.span, diff --git a/src/flatten.rs b/src/flatten.rs index f994f6778c..86b97f2233 100644 --- a/src/flatten.rs +++ b/src/flatten.rs @@ -3,6 +3,7 @@ use crate::{Block, Expr, Expression, ParserWorkingSet, Pipeline, Span, Statement #[derive(Debug)] pub enum FlatShape { Garbage, + Bool, Int, InternalCall, External, @@ -57,6 +58,10 @@ impl<'a> ParserWorkingSet<'a> { Expr::Int(_) => { vec![(expr.span, FlatShape::Int)] } + Expr::Bool(_) => { + vec![(expr.span, FlatShape::Bool)] + } + Expr::List(list) => { let mut output = vec![]; for l in list { @@ -64,9 +69,7 @@ impl<'a> ParserWorkingSet<'a> { } output } - Expr::Literal(_) => { - vec![(expr.span, FlatShape::Literal)] - } + Expr::Keyword(_, expr) => self.flatten_expression(expr), Expr::Operator(_) => { vec![(expr.span, FlatShape::Operator)] } diff --git a/src/main.rs b/src/main.rs index 210859f7c8..ef90257728 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,33 +15,30 @@ fn main() -> std::io::Result<()> { working_set.add_decl(sig.into()); let sig = Signature::build("if") - .required("cond", SyntaxShape::RowCondition, "condition") + .required("cond", SyntaxShape::Expression, "condition") .required("then_block", SyntaxShape::Block, "then block") - .required( + .optional( "else", - SyntaxShape::Literal(b"else".to_vec()), - "else keyword", - ) - .required("else_block", SyntaxShape::Block, "else block"); + SyntaxShape::Keyword(b"else".to_vec(), Box::new(SyntaxShape::Block)), + "optional else followed by else block", + ); working_set.add_decl(sig.into()); let sig = Signature::build("let") .required("var_name", SyntaxShape::Variable, "variable name") - .required("=", SyntaxShape::Literal(b"=".to_vec()), "equals sign") .required( - "value", - SyntaxShape::Expression, - "the value to set the variable to", + "initial_value", + SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), + "equals sign followed by value", ); working_set.add_decl(sig.into()); let sig = Signature::build("alias") .required("var_name", SyntaxShape::Variable, "variable name") - .required("=", SyntaxShape::Literal(b"=".to_vec()), "equals sign") .required( - "value", - SyntaxShape::Expression, - "the value to set the variable to", + "initial_value", + SyntaxShape::Keyword(b"=".to_vec(), Box::new(SyntaxShape::Expression)), + "equals sign followed by value", ); working_set.add_decl(sig.into()); @@ -161,7 +158,7 @@ fn main() -> std::io::Result<()> { s.as_bytes(), false, ); - // println!("{:?}", output); + println!("{:?}", output); if let Some(err) = err { println!("Error: {:?}", err); continue; diff --git a/src/parser.rs b/src/parser.rs index d5b340ccf1..10bb480b67 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -11,7 +11,7 @@ use crate::{ #[derive(Debug, Clone, PartialEq, Eq)] pub enum SyntaxShape { /// A specific match to a word or symbol - Literal(Vec), + Keyword(Vec, Box), /// Any syntactic form is allowed Any, @@ -95,7 +95,7 @@ impl SyntaxShape { let contents = x.to_type(); Type::List(Box::new(contents)) } - SyntaxShape::Literal(..) => Type::Unknown, + SyntaxShape::Keyword(_, expr) => expr.to_type(), SyntaxShape::MathExpression => Type::Unknown, SyntaxShape::Number => Type::Number, SyntaxShape::Operator => Type::Unknown, @@ -160,6 +160,7 @@ impl Call { #[derive(Debug, Clone)] pub enum Expr { + Bool(bool), Int(i64), Var(VarId), Call(Box), @@ -170,7 +171,7 @@ pub enum Expr { Block(BlockId), List(Vec), Table(Vec, Vec>), - Literal(Vec), + Keyword(Vec, Box), String(String), // FIXME: improve this in the future? Signature(Signature), Garbage, @@ -238,6 +239,13 @@ impl Expression { } } + pub fn as_keyword(&self) -> Option<&Expression> { + match &self.expr { + Expr::Keyword(_, expr) => Some(expr), + _ => None, + } + } + pub fn as_var(&self) -> Option { match self.expr { Expr::Var(var_id) => Some(var_id), @@ -419,11 +427,11 @@ impl<'a> ParserWorkingSet<'a> { // and we also have the argument let mut span = arg_span; span.start += long_name.len() + 1; //offset by long flag and '=' - let (arg, err) = self.parse_value(span, arg_shape.clone()); + let (arg, err) = self.parse_value(span, arg_shape); (Some(long_name), Some(arg), err) } else if let Some(arg) = spans.get(*spans_idx + 1) { - let (arg, err) = self.parse_value(*arg, arg_shape.clone()); + let (arg, err) = self.parse_value(*arg, arg_shape); *spans_idx += 1; (Some(long_name), Some(arg), err) @@ -520,11 +528,76 @@ impl<'a> ParserWorkingSet<'a> { } } + fn calculate_end_span( + &self, + decl: &Declaration, + spans: &[Span], + spans_idx: usize, + positional_idx: usize, + ) -> usize { + if decl.signature.rest_positional.is_some() { + spans.len() + } else { + // println!("num_positionals: {}", decl.signature.num_positionals()); + // println!("positional_idx: {}", positional_idx); + // println!("spans.len(): {}", spans.len()); + // println!("spans_idx: {}", spans_idx); + + // check to see if a keyword follows the current position. + + let mut next_keyword_idx = spans.len(); + for idx in (positional_idx + 1)..decl.signature.num_positionals() { + match decl.signature.get_positional(idx) { + Some(PositionalArg { + shape: SyntaxShape::Keyword(kw, ..), + .. + }) => { + for span_idx in spans_idx..spans.len() { + let contents = self.get_span_contents(spans[span_idx]); + + if contents == kw { + next_keyword_idx = span_idx - (idx - (positional_idx + 1)); + break; + } + } + } + _ => {} + } + } + + let remainder = decl.signature.num_positionals_after(positional_idx); + let remainder_idx = if remainder < spans.len() { + spans.len() - remainder + 1 + } else { + spans_idx + 1 + }; + + let end = [next_keyword_idx, remainder_idx, spans.len()] + .iter() + .min() + .expect("internal error: can't find min") + .clone(); + + // println!( + // "{:?}", + // [ + // next_keyword_idx, + // remainder_idx, + // spans.len(), + // spans_idx, + // remainder, + // positional_idx, + // ] + // ); + end + } + } + fn parse_multispan_value( &mut self, spans: &[Span], spans_idx: &mut usize, - shape: SyntaxShape, + shape: &SyntaxShape, ) -> (Expression, Option) { let mut error = None; @@ -538,46 +611,69 @@ impl<'a> ParserWorkingSet<'a> { SyntaxShape::RowCondition => { let (arg, err) = self.parse_row_condition(&spans[*spans_idx..]); error = error.or(err); - *spans_idx = spans.len(); + *spans_idx = spans.len() - 1; (arg, error) } SyntaxShape::Expression => { let (arg, err) = self.parse_expression(&spans[*spans_idx..]); error = error.or(err); - *spans_idx = spans.len(); + *spans_idx = spans.len() - 1; (arg, error) } - SyntaxShape::Literal(literal) => { + SyntaxShape::Keyword(keyword, arg) => { let arg_span = spans[*spans_idx]; let arg_contents = self.get_span_contents(arg_span); - if arg_contents != literal { + if arg_contents != keyword { // When keywords mismatch, this is a strong indicator of something going wrong. // We won't often override the current error, but as this is a strong indicator // go ahead and override the current error and tell the user about the missing // keyword/literal. error = Some(ParseError::Mismatch( - String::from_utf8_lossy(&literal).into(), + String::from_utf8_lossy(&keyword).into(), arg_span, )) } + + *spans_idx += 1; + if *spans_idx >= spans.len() { + error = error.or(Some(ParseError::MissingPositional( + String::from_utf8_lossy(&keyword).into(), + spans[*spans_idx - 1], + ))); + return ( + Expression { + expr: Expr::Keyword( + keyword.clone(), + Box::new(Expression::garbage(arg_span)), + ), + span: arg_span, + ty: Type::Unknown, + }, + error, + ); + } + let (expr, err) = self.parse_multispan_value(&spans, spans_idx, arg); + error = error.or(err); + let ty = expr.ty.clone(); + ( Expression { - expr: Expr::Literal(literal), + expr: Expr::Keyword(keyword.clone(), Box::new(expr)), span: arg_span, - ty: Type::Unknown, + ty, }, error, ) } - _ => { + x => { // All other cases are single-span values let arg_span = spans[*spans_idx]; - let (arg, err) = self.parse_value(arg_span, shape); + let (arg, err) = self.parse_value(arg_span, &shape); error = error.or(err); (arg, error) @@ -629,7 +725,7 @@ impl<'a> ParserWorkingSet<'a> { for flag in short_flags { if let Some(arg_shape) = flag.arg { if let Some(arg) = spans.get(spans_idx + 1) { - let (arg, err) = self.parse_value(*arg, arg_shape.clone()); + let (arg, err) = self.parse_value(*arg, &arg_shape); error = error.or(err); call.named.push((flag.long.clone(), Some(arg))); @@ -649,35 +745,17 @@ impl<'a> ParserWorkingSet<'a> { if let Some(positional) = decl.signature.get_positional(positional_idx) { //Make sure we leave enough spans for the remaining positionals - let end = if decl.signature.rest_positional.is_some() { - spans.len() - } else { - // println!("num_positionals: {}", decl.signature.num_positionals()); - // println!("positional_idx: {}", positional_idx); - // println!("spans.len(): {}", spans.len()); - // println!("spans_idx: {}", spans_idx); - let remainder = decl.signature.num_positionals() - positional_idx; - - if remainder >= spans.len() { - spans.len() - } else { - spans.len() - remainder + 1 - } - }; - // println!("end: {}", end); + let end = self.calculate_end_span(&decl, spans, spans_idx, positional_idx); let orig_idx = spans_idx; - let (arg, err) = self.parse_multispan_value( - &spans[..end], - &mut spans_idx, - positional.shape.clone(), - ); + let (arg, err) = + self.parse_multispan_value(&spans[..end], &mut spans_idx, &positional.shape); error = error.or(err); let arg = if positional.shape.to_type() != Type::Unknown && arg.ty != positional.shape.to_type() { - let span = span(&spans[orig_idx..spans_idx + 1]); + let span = span(&spans[orig_idx..spans_idx]); error = error.or(Some(ParseError::TypeMismatch( positional.shape.to_type(), span, @@ -819,23 +897,32 @@ impl<'a> ParserWorkingSet<'a> { } pub(crate) fn parse_dollar_expr(&mut self, span: Span) -> (Expression, Option) { - let bytes = self.get_span_contents(span); - - if let Some(var_id) = self.find_variable(bytes) { - ( - Expression { - expr: Expr::Var(var_id), - span, - ty: self.get_variable(var_id).clone(), - }, - None, - ) - } else { - (garbage(span), Some(ParseError::VariableNotFound(span))) - } + self.parse_variable_expr(span) } pub fn parse_variable_expr(&mut self, span: Span) -> (Expression, Option) { + let contents = self.get_span_contents(span); + + if contents == b"$true" { + return ( + Expression { + expr: Expr::Bool(true), + span, + ty: Type::Bool, + }, + None, + ); + } else if contents == b"$false" { + return ( + Expression { + expr: Expr::Bool(false), + span, + ty: Type::Bool, + }, + None, + ); + } + let (id, err) = self.parse_variable(span); if err.is_none() { @@ -849,17 +936,9 @@ impl<'a> ParserWorkingSet<'a> { None, ) } else { - let name = self.get_span_contents(span).to_vec(); - // this seems okay to set it to unknown here, but we should double-check - let id = self.add_variable(name, Type::Unknown); - ( - Expression { - expr: Expr::Var(id), - span, - ty: Type::Unknown, - }, - None, + Expression::garbage(span), + Some(ParseError::VariableNotFound(span)), ) } } else { @@ -1402,11 +1481,8 @@ impl<'a> ParserWorkingSet<'a> { let mut spans_idx = 0; while spans_idx < arg.parts.len() { - let (arg, err) = self.parse_multispan_value( - &arg.parts, - &mut spans_idx, - element_shape.clone(), - ); + let (arg, err) = + self.parse_multispan_value(&arg.parts, &mut spans_idx, element_shape); error = error.or(err); args.push(arg); @@ -1477,7 +1553,7 @@ impl<'a> ParserWorkingSet<'a> { let mut table_headers = vec![]; let (headers, err) = - self.parse_value(output.block[0].commands[0].parts[0], SyntaxShape::Table); + self.parse_value(output.block[0].commands[0].parts[0], &SyntaxShape::Table); error = error.or(err); if let Expression { @@ -1490,7 +1566,7 @@ impl<'a> ParserWorkingSet<'a> { let mut rows = vec![]; for part in &output.block[1].commands[0].parts { - let (values, err) = self.parse_value(*part, SyntaxShape::Table); + let (values, err) = self.parse_value(*part, &SyntaxShape::Table); error = error.or(err); if let Expression { expr: Expr::List(values), @@ -1570,7 +1646,7 @@ impl<'a> ParserWorkingSet<'a> { pub fn parse_value( &mut self, span: Span, - shape: SyntaxShape, + shape: &SyntaxShape, ) -> (Expression, Option) { let bytes = self.get_span_contents(span); @@ -1580,7 +1656,7 @@ impl<'a> ParserWorkingSet<'a> { // We check variable first because immediately following we check for variables with column paths // which might result in a value that fits other shapes (and require the variable to already be // declared) - if shape == SyntaxShape::Variable { + if shape == &SyntaxShape::Variable { return self.parse_variable_expr(span); } else if bytes.starts_with(b"$") { return self.parse_dollar_expr(span); @@ -1622,26 +1698,6 @@ impl<'a> ParserWorkingSet<'a> { ) } } - SyntaxShape::Literal(literal) => { - if bytes == literal { - ( - Expression { - expr: Expr::Literal(literal), - span, - ty: Type::Unknown, - }, - None, - ) - } else { - ( - garbage(span), - Some(ParseError::Mismatch( - format!("keyword '{}'", String::from_utf8_lossy(&literal)), - span, - )), - ) - } - } SyntaxShape::String | SyntaxShape::GlobPattern | SyntaxShape::FilePath => { self.parse_string(span) } @@ -1698,7 +1754,7 @@ impl<'a> ParserWorkingSet<'a> { SyntaxShape::String, ]; for shape in shapes.iter() { - if let (s, None) = self.parse_value(span, shape.clone()) { + if let (s, None) = self.parse_value(span, shape) { return (s, None); } } @@ -1771,7 +1827,7 @@ impl<'a> ParserWorkingSet<'a> { let mut last_prec = 1000000; let mut error = None; - let (lhs, err) = self.parse_value(spans[0], SyntaxShape::Any); + let (lhs, err) = self.parse_value(spans[0], &SyntaxShape::Any); error = error.or(err); idx += 1; @@ -1795,7 +1851,7 @@ impl<'a> ParserWorkingSet<'a> { break; } - let (rhs, err) = self.parse_value(spans[idx], SyntaxShape::Any); + let (rhs, err) = self.parse_value(spans[idx], &SyntaxShape::Any); error = error.or(err); if op_prec <= last_prec { diff --git a/src/signature.rs b/src/signature.rs index 0633dc3be9..8d1ce73287 100644 --- a/src/signature.rs +++ b/src/signature.rs @@ -200,7 +200,66 @@ impl Signature { } pub fn num_positionals(&self) -> usize { - self.required_positional.len() + self.optional_positional.len() + let mut total = self.required_positional.len() + self.optional_positional.len(); + + for positional in &self.required_positional { + match positional.shape { + SyntaxShape::Keyword(..) => { + // Keywords have a required argument, so account for that + total += 1; + } + _ => {} + } + } + for positional in &self.optional_positional { + match positional.shape { + SyntaxShape::Keyword(..) => { + // Keywords have a required argument, so account for that + total += 1; + } + _ => {} + } + } + total + } + + pub fn num_positionals_after(&self, idx: usize) -> usize { + let mut total = 0; + let mut curr = 0; + + for positional in &self.required_positional { + match positional.shape { + SyntaxShape::Keyword(..) => { + // Keywords have a required argument, so account for that + if curr > idx { + total += 2; + } + } + _ => { + if curr > idx { + total += 1; + } + } + } + curr += 1; + } + for positional in &self.optional_positional { + match positional.shape { + SyntaxShape::Keyword(..) => { + // Keywords have a required argument, so account for that + if curr > idx { + total += 2; + } + } + _ => { + if curr > idx { + total += 1; + } + } + } + curr += 1; + } + total } /// Find the matching long flag diff --git a/src/syntax_highlight.rs b/src/syntax_highlight.rs index 5474479ced..197f65ba6d 100644 --- a/src/syntax_highlight.rs +++ b/src/syntax_highlight.rs @@ -54,6 +54,9 @@ impl Highlighter for NuHighlighter { FlatShape::Int => { output.push((Style::new().fg(nu_ansi_term::Color::Green), next_token)) } + FlatShape::Bool => { + output.push((Style::new().fg(nu_ansi_term::Color::LightCyan), next_token)) + } FlatShape::Literal => { output.push((Style::new().fg(nu_ansi_term::Color::Blue), next_token)) } From ad48387aa0536cb40a3b2849363d90ef52f46aac Mon Sep 17 00:00:00 2001 From: JT Date: Sat, 24 Jul 2021 18:44:38 +1200 Subject: [PATCH 2/2] WIP --- src/eval.rs | 31 +++++++++++++++++++++++++++++++ src/main.rs | 2 +- src/parser.rs | 3 +++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/eval.rs b/src/eval.rs index 30bf276fc6..29d62824ab 100644 --- a/src/eval.rs +++ b/src/eval.rs @@ -103,6 +103,37 @@ fn eval_call(state: &State, stack: &mut Stack, call: &Call) -> Result { + if val { + let block = state.parser_state.get_block(then_block); + eval_block(state, stack, block) + } else if let Some(else_case) = else_case { + println!("{:?}", else_case); + if let Some(else_expr) = else_case.as_keyword() { + if let Some(block_id) = else_expr.as_block() { + let block = state.parser_state.get_block(block_id); + eval_block(state, stack, block) + } else { + eval_expression(state, stack, else_expr) + } + } else { + eval_expression(state, stack, else_case) + } + } else { + Ok(Value::Unknown) + } + } + _ => Err(ShellError::Mismatch("bool".into(), Span::unknown())), + } } else { Ok(Value::Unknown) } diff --git a/src/main.rs b/src/main.rs index ef90257728..04a4a07128 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,7 +19,7 @@ fn main() -> std::io::Result<()> { .required("then_block", SyntaxShape::Block, "then block") .optional( "else", - SyntaxShape::Keyword(b"else".to_vec(), Box::new(SyntaxShape::Block)), + SyntaxShape::Keyword(b"else".to_vec(), Box::new(SyntaxShape::Expression)), "optional else followed by else block", ); working_set.add_decl(sig.into()); diff --git a/src/parser.rs b/src/parser.rs index 10bb480b67..d0bf685134 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -936,6 +936,9 @@ impl<'a> ParserWorkingSet<'a> { None, ) } else { + let name = self.get_span_contents(span).to_vec(); + // this seems okay to set it to unknown here, but we should double-check + let id = self.add_variable(name, Type::Unknown); ( Expression::garbage(span), Some(ParseError::VariableNotFound(span)),