From 1ad9d6f199556c706812d992f22a7a91e8668259 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Tue, 17 Sep 2019 15:26:27 -0700 Subject: [PATCH 1/2] Overhaul the expansion system The main thrust of this (very large) commit is an overhaul of the expansion system. The parsing pipeline is: - Lightly parse the source file for atoms, basic delimiters and pipeline structure into a token tree - Expand the token tree into a HIR (high-level intermediate representation) based upon the baseline syntax rules for expressions and the syntactic shape of commands. Somewhat non-traditionally, nu doesn't have an AST at all. It goes directly from the token tree, which doesn't represent many important distinctions (like the difference between `hello` and `5KB`) directly into a high-level representation that doesn't have a direct correspondence to the source code. At a high level, nu commands work like macros, in the sense that the syntactic shape of the invocation of a command depends on the definition of a command. However, commands do not have the ability to perform unrestricted expansions of the token tree. Instead, they describe their arguments in terms of syntactic shapes, and the expander expands the token tree into HIR based upon that definition. For example, the `where` command says that it takes a block as its first required argument, and the description of the block syntactic shape expands the syntax `cpu > 10` into HIR that represents `{ $it.cpu > 10 }`. This commit overhauls that system so that the syntactic shapes are described in terms of a few new traits (`ExpandSyntax` and `ExpandExpression` are the primary ones) that are more composable than the previous system. The first big win of this new system is the addition of the `ColumnPath` shape, which looks like `cpu."max ghz"` or `package.version`. Previously, while a variable path could look like `$it.cpu."max ghz"`, the tail of a variable path could not be easily reused in other contexts. Now, that tail is its own syntactic shape, and it can be used as part of a command's signature. This cleans up commands like `inc`, `add` and `edit` as well as shorthand blocks, which can now look like `| where cpu."max ghz" > 10` --- Cargo.lock | 22 + Cargo.toml | 3 + src/cli.rs | 190 +- src/commands.rs | 1 + src/commands/autoview.rs | 33 +- src/commands/classified.rs | 35 +- src/commands/command.rs | 9 + src/commands/echo.rs | 9 +- src/commands/enter.rs | 10 +- src/commands/fetch.rs | 2 +- src/commands/first.rs | 2 +- src/commands/get.rs | 61 +- src/commands/open.rs | 2 +- src/commands/save.rs | 7 +- src/commands/skip_while.rs | 3 + src/commands/tags.rs | 4 +- src/context.rs | 24 +- src/data/base.rs | 133 +- src/data/meta.rs | 123 +- src/errors.rs | 31 +- src/evaluate/evaluator.rs | 30 +- src/lib.rs | 2 +- src/parser.rs | 6 +- src/parser/deserializer.rs | 9 +- src/parser/hir.rs | 138 +- src/parser/hir/baseline_parse.rs | 142 +- src/parser/hir/baseline_parse/tests.rs | 144 ++ src/parser/hir/baseline_parse_tokens.rs | 459 ----- src/parser/hir/binary.rs | 6 + src/parser/hir/expand_external_tokens.rs | 87 + src/parser/hir/external_command.rs | 2 +- src/parser/hir/path.rs | 34 +- src/parser/hir/syntax_shape.rs | 662 +++++++ src/parser/hir/syntax_shape/block.rs | 168 ++ src/parser/hir/syntax_shape/expression.rs | 188 ++ .../hir/syntax_shape/expression/delimited.rs | 38 + .../hir/syntax_shape/expression/file_path.rs | 59 + .../hir/syntax_shape/expression/list.rs | 43 + .../hir/syntax_shape/expression/number.rs | 97 + .../hir/syntax_shape/expression/pattern.rs | 86 + .../hir/syntax_shape/expression/string.rs | 60 + .../hir/syntax_shape/expression/unit.rs | 89 + .../syntax_shape/expression/variable_path.rs | 396 ++++ src/parser/hir/tokens_iterator.rs | 365 ++++ src/parser/hir/tokens_iterator/debug.rs | 30 + src/parser/parse/files.rs | 29 +- src/parser/parse/operator.rs | 3 + src/parser/parse/parser.rs | 1586 ++++++++--------- src/parser/parse/pipeline.rs | 27 +- src/parser/parse/token_tree.rs | 112 +- src/parser/parse/token_tree_builder.rs | 110 +- src/parser/parse/tokens.rs | 11 +- src/parser/parse_command.rs | 155 +- src/parser/registry.rs | 19 +- src/plugins/add.rs | 23 +- src/plugins/edit.rs | 18 +- src/plugins/inc.rs | 37 +- src/plugins/str.rs | 47 +- src/shell/helper.rs | 38 +- tests/command_open_tests.rs | 2 +- tests/helpers/mod.rs | 1 + 61 files changed, 4310 insertions(+), 1952 deletions(-) create mode 100644 src/parser/hir/baseline_parse/tests.rs delete mode 100644 src/parser/hir/baseline_parse_tokens.rs create mode 100644 src/parser/hir/expand_external_tokens.rs create mode 100644 src/parser/hir/syntax_shape.rs create mode 100644 src/parser/hir/syntax_shape/block.rs create mode 100644 src/parser/hir/syntax_shape/expression.rs create mode 100644 src/parser/hir/syntax_shape/expression/delimited.rs create mode 100644 src/parser/hir/syntax_shape/expression/file_path.rs create mode 100644 src/parser/hir/syntax_shape/expression/list.rs create mode 100644 src/parser/hir/syntax_shape/expression/number.rs create mode 100644 src/parser/hir/syntax_shape/expression/pattern.rs create mode 100644 src/parser/hir/syntax_shape/expression/string.rs create mode 100644 src/parser/hir/syntax_shape/expression/unit.rs create mode 100644 src/parser/hir/syntax_shape/expression/variable_path.rs create mode 100644 src/parser/hir/tokens_iterator.rs create mode 100644 src/parser/hir/tokens_iterator/debug.rs diff --git a/Cargo.lock b/Cargo.lock index 852fbd6103..af1d46aa0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1491,6 +1491,25 @@ dependencies = [ "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "nom-tracable" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nom-tracable-macros 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nom_locate 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nom-tracable-macros" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "nom_locate" version = "1.0.0" @@ -1550,6 +1569,7 @@ dependencies = [ "natural 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "neso 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nom-tracable 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "nom_locate 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "num-bigint 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3140,6 +3160,8 @@ dependencies = [ "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" "checksum nom 5.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e9761d859320e381010a4f7f8ed425f2c924de33ad121ace447367c713ad561b" +"checksum nom-tracable 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "edaa64ad2837d831d4a17966c9a83aa5101cc320730f5b724811c8f7442a2528" +"checksum nom-tracable-macros 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fd25f70877a9fe68bd406b3dd3ff99e94ce9de776cf2a96e0d99de90b53d4765" "checksum nom_locate 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f932834fd8e391fc7710e2ba17e8f9f8645d846b55aa63207e17e110a1e1ce35" "checksum ntapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f26e041cd983acbc087e30fcba770380cfa352d0e392e175b2344ebaf7ea0602" "checksum num-bigint 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f9c3f34cdd24f334cb265d9bf8bfa8a241920d026916785747a92f0e55541a1a" diff --git a/Cargo.toml b/Cargo.toml index f51ea06d8a..66bd695c08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ surf = "1.0.2" url = "2.1.0" roxmltree = "0.7.0" nom_locate = "1.0.0" +nom-tracable = "0.4.0" enum-utils = "0.1.1" unicode-xid = "0.2.0" serde_ini = "0.2.0" @@ -95,6 +96,8 @@ textview = ["syntect", "onig_sys", "crossterm"] binaryview = ["image", "crossterm"] sys = ["heim", "battery"] ps = ["heim"] +trace = ["nom-tracable/trace"] +all = ["raw-key", "textview", "binaryview", "sys", "ps", "clipboard", "ptree"] [dependencies.rusqlite] version = "0.20.0" diff --git a/src/cli.rs b/src/cli.rs index 38e2474faf..6a35608d91 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,4 +1,3 @@ -use crate::commands::autoview; use crate::commands::classified::{ ClassifiedCommand, ClassifiedInputStream, ClassifiedPipeline, ExternalCommand, InternalCommand, StreamNext, @@ -13,7 +12,12 @@ pub(crate) use crate::errors::ShellError; use crate::fuzzysearch::{interactive_fuzzy_search, SelectionResult}; use crate::git::current_branch; use crate::parser::registry::Signature; -use crate::parser::{hir, CallNode, Pipeline, PipelineElement, TokenNode}; +use crate::parser::{ + hir, + hir::syntax_shape::{CommandHeadShape, CommandSignature, ExpandSyntax}, + hir::{expand_external_tokens::expand_external_tokens, tokens_iterator::TokensIterator}, + parse_command_tail, Pipeline, PipelineElement, TokenNode, +}; use crate::prelude::*; use log::{debug, trace}; @@ -25,6 +29,7 @@ use std::io::{BufRead, BufReader, Write}; use std::iter::Iterator; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; #[derive(Debug)] pub enum MaybeOwned<'a, T> { @@ -75,7 +80,7 @@ fn load_plugin(path: &std::path::Path, context: &mut Context) -> Result<(), Shel let name = params.name.clone(); let fname = fname.to_string(); - if context.has_command(&name) { + if let Some(_) = context.get_command(&name) { trace!("plugin {:?} already loaded.", &name); } else { if params.is_filter { @@ -428,21 +433,11 @@ pub async fn cli() -> Result<(), Box> { } } - LineResult::Error(mut line, err) => { + LineResult::Error(line, err) => { rl.add_history_entry(line.clone()); - let diag = err.to_diagnostic(); + context.with_host(|host| { - let writer = host.err_termcolor(); - line.push_str(" "); - let files = crate::parser::Files::new(line); - let _ = std::panic::catch_unwind(move || { - let _ = language_reporting::emit( - &mut writer.lock(), - &files, - &diag, - &language_reporting::DefaultConfig, - ); - }); + print_err(err, host, &Text::from(line)); }) } @@ -459,6 +454,14 @@ pub async fn cli() -> Result<(), Box> { Ok(()) } +fn chomp_newline(s: &str) -> &str { + if s.ends_with('\n') { + &s[..s.len() - 1] + } else { + s + } +} + enum LineResult { Success(String), Error(String, ShellError), @@ -471,9 +474,11 @@ async fn process_line(readline: Result, ctx: &mut Context Ok(line) if line.trim() == "" => LineResult::Success(line.clone()), Ok(line) => { - let result = match crate::parser::parse(&line, uuid::Uuid::nil()) { + let line = chomp_newline(line); + + let result = match crate::parser::parse(&line, uuid::Uuid::new_v4()) { Err(err) => { - return LineResult::Error(line.clone(), err); + return LineResult::Error(line.to_string(), err); } Ok(val) => val, @@ -484,7 +489,7 @@ async fn process_line(readline: Result, ctx: &mut Context let mut pipeline = match classify_pipeline(&result, ctx, &Text::from(line)) { Ok(pipeline) => pipeline, - Err(err) => return LineResult::Error(line.clone(), err), + Err(err) => return LineResult::Error(line.to_string(), err), }; match pipeline.commands.last() { @@ -492,7 +497,7 @@ async fn process_line(readline: Result, ctx: &mut Context _ => pipeline .commands .push(ClassifiedCommand::Internal(InternalCommand { - command: whole_stream_command(autoview::Autoview), + name: "autoview".to_string(), name_tag: Tag::unknown(), args: hir::Call::new( Box::new(hir::Expression::synthetic_string("autoview")), @@ -514,16 +519,24 @@ async fn process_line(readline: Result, ctx: &mut Context input = match (item, next) { (None, _) => break, + (Some(ClassifiedCommand::Dynamic(_)), _) + | (_, Some(ClassifiedCommand::Dynamic(_))) => { + return LineResult::Error( + line.to_string(), + ShellError::unimplemented("Dynamic commands"), + ) + } + (Some(ClassifiedCommand::Expr(_)), _) => { return LineResult::Error( - line.clone(), + line.to_string(), ShellError::unimplemented("Expression-only commands"), ) } (_, Some(ClassifiedCommand::Expr(_))) => { return LineResult::Error( - line.clone(), + line.to_string(), ShellError::unimplemented("Expression-only commands"), ) } @@ -536,7 +549,7 @@ async fn process_line(readline: Result, ctx: &mut Context .await { Ok(val) => ClassifiedInputStream::from_input_stream(val), - Err(err) => return LineResult::Error(line.clone(), err), + Err(err) => return LineResult::Error(line.to_string(), err), }, (Some(ClassifiedCommand::Internal(left)), Some(_)) => { @@ -545,7 +558,7 @@ async fn process_line(readline: Result, ctx: &mut Context .await { Ok(val) => ClassifiedInputStream::from_input_stream(val), - Err(err) => return LineResult::Error(line.clone(), err), + Err(err) => return LineResult::Error(line.to_string(), err), } } @@ -555,7 +568,7 @@ async fn process_line(readline: Result, ctx: &mut Context .await { Ok(val) => ClassifiedInputStream::from_input_stream(val), - Err(err) => return LineResult::Error(line.clone(), err), + Err(err) => return LineResult::Error(line.to_string(), err), } } @@ -564,20 +577,20 @@ async fn process_line(readline: Result, ctx: &mut Context Some(ClassifiedCommand::External(_)), ) => match left.run(ctx, input, StreamNext::External).await { Ok(val) => val, - Err(err) => return LineResult::Error(line.clone(), err), + Err(err) => return LineResult::Error(line.to_string(), err), }, (Some(ClassifiedCommand::External(left)), Some(_)) => { match left.run(ctx, input, StreamNext::Internal).await { Ok(val) => val, - Err(err) => return LineResult::Error(line.clone(), err), + Err(err) => return LineResult::Error(line.to_string(), err), } } (Some(ClassifiedCommand::External(left)), None) => { match left.run(ctx, input, StreamNext::Last).await { Ok(val) => val, - Err(err) => return LineResult::Error(line.clone(), err), + Err(err) => return LineResult::Error(line.to_string(), err), } } }; @@ -585,7 +598,7 @@ async fn process_line(readline: Result, ctx: &mut Context is_first_command = false; } - LineResult::Success(line.clone()) + LineResult::Success(line.to_string()) } Err(ReadlineError::Interrupted) => LineResult::CtrlC, Err(ReadlineError::Eof) => LineResult::Break, @@ -616,80 +629,91 @@ fn classify_pipeline( } fn classify_command( - command: &PipelineElement, + command: &Tagged, context: &Context, source: &Text, ) -> Result { - let call = command.call(); + let mut iterator = TokensIterator::new(&command.tokens.item, command.tag, true); + + let head = CommandHeadShape + .expand_syntax(&mut iterator, &context.expand_context(source, command.tag))?; + + match &head { + CommandSignature::Expression(_) => Err(ShellError::syntax_error( + "Unexpected expression in command position".tagged(command.tag), + )), - match call { // If the command starts with `^`, treat it as an external command no matter what - call if call.head().is_external() => { - let name_tag = call.head().expect_external(); - let name = name_tag.slice(source); + CommandSignature::External(name) => { + let name_str = name.slice(source); - Ok(external_command(call, source, name.tagged(name_tag))) + external_command(&mut iterator, source, name_str.tagged(name)) } - // Otherwise, if the command is a bare word, we'll need to triage it - call if call.head().is_bare() => { - let head = call.head(); - let name = head.source(source); + CommandSignature::LiteralExternal { outer, inner } => { + let name_str = inner.slice(source); - match context.has_command(name) { - // if the command is in the registry, it's an internal command - true => { - let command = context.get_command(name); - let config = command.signature(); - - trace!(target: "nu::build_pipeline", "classifying {:?}", config); - - let args: hir::Call = config.parse_args(call, &context, source)?; - - trace!(target: "nu::build_pipeline", "args :: {}", args.debug(source)); - - Ok(ClassifiedCommand::Internal(InternalCommand { - command, - name_tag: head.tag(), - args, - })) - } - - // otherwise, it's an external command - false => Ok(external_command(call, source, name.tagged(head.tag()))), - } + external_command(&mut iterator, source, name_str.tagged(outer)) } - // If the command is something else (like a number or a variable), that is currently unsupported. - // We might support `$somevar` as a curried command in the future. - call => Err(ShellError::invalid_command(call.head().tag())), + CommandSignature::Internal(command) => { + let tail = parse_command_tail( + &command.signature(), + &context.expand_context(source, command.tag), + &mut iterator, + command.tag, + )?; + + let (positional, named) = match tail { + None => (None, None), + Some((positional, named)) => (positional, named), + }; + + let call = hir::Call { + head: Box::new(head.to_expression()), + positional, + named, + }; + + Ok(ClassifiedCommand::Internal(InternalCommand::new( + command.name().to_string(), + command.tag, + call, + ))) + } } } // Classify this command as an external command, which doesn't give special meaning // to nu syntactic constructs, and passes all arguments to the external command as // strings. -fn external_command( - call: &Tagged, +pub(crate) fn external_command( + tokens: &mut TokensIterator, source: &Text, name: Tagged<&str>, -) -> ClassifiedCommand { - let arg_list_strings: Vec> = match call.children() { - Some(args) => args - .iter() - .filter_map(|i| match i { - TokenNode::Whitespace(_) => None, - other => Some(other.as_external_arg(source).tagged(other.tag())), - }) - .collect(), - None => vec![], - }; +) -> Result { + let arg_list_strings = expand_external_tokens(tokens, source)?; - let (name, tag) = name.into_parts(); - - ClassifiedCommand::External(ExternalCommand { + Ok(ClassifiedCommand::External(ExternalCommand { name: name.to_string(), - name_tag: tag, + name_tag: name.tag(), args: arg_list_strings, - }) + })) +} + +pub fn print_err(err: ShellError, host: &dyn Host, source: &Text) { + let diag = err.to_diagnostic(); + + let writer = host.err_termcolor(); + let mut source = source.to_string(); + source.push_str(" "); + let files = crate::parser::Files::new(source); + let _ = std::panic::catch_unwind(move || { + let _ = language_reporting::emit( + &mut writer.lock(), + &files, + &diag, + &language_reporting::DefaultConfig, + ); + }); } diff --git a/src/commands.rs b/src/commands.rs index 72c07e38e6..4eb733edd4 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -75,6 +75,7 @@ pub(crate) use command::{ UnevaluatedCallInfo, WholeStreamCommand, }; +pub(crate) use classified::ClassifiedCommand; pub(crate) use config::Config; pub(crate) use cp::Cpy; pub(crate) use date::Date; diff --git a/src/commands/autoview.rs b/src/commands/autoview.rs index a0e7e9a8a5..57ab6269b3 100644 --- a/src/commands/autoview.rs +++ b/src/commands/autoview.rs @@ -58,21 +58,21 @@ pub fn autoview( } } }; - } else if is_single_anchored_text_value(&input) { - let text = context.get_command("textview"); - if let Some(text) = text { - let result = text.run(raw.with_input(input), &context.commands, false); - result.collect::>().await; - } else { - for i in input { - match i.item { - Value::Primitive(Primitive::String(s)) => { - println!("{}", s); - } - _ => {} - } - } - } + // } else if is_single_origined_text_value(&input) { + // let text = context.get_command("textview"); + // if let Some(text) = text { + // let result = text.run(raw.with_input(input), &context.commands); + // result.collect::>().await; + // } else { + // for i in input { + // match i.item { + // Value::Primitive(Primitive::String(s)) => { + // println!("{}", s); + // } + // _ => {} + // } + // } + // } } else if is_single_text_value(&input) { for i in input { match i.item { @@ -111,7 +111,8 @@ fn is_single_text_value(input: &Vec>) -> bool { } } -fn is_single_anchored_text_value(input: &Vec>) -> bool { +#[allow(unused)] +fn is_single_origined_text_value(input: &Vec>) -> bool { if input.len() != 1 { return false; } diff --git a/src/commands/classified.rs b/src/commands/classified.rs index 0e5cd95d8d..d30025b944 100644 --- a/src/commands/classified.rs +++ b/src/commands/classified.rs @@ -1,12 +1,11 @@ -use crate::commands::Command; use crate::parser::{hir, TokenNode}; use crate::prelude::*; use bytes::{BufMut, BytesMut}; +use derive_new::new; use futures::stream::StreamExt; use futures_codec::{Decoder, Encoder, Framed}; use log::{log_enabled, trace}; use std::io::{Error, ErrorKind}; -use std::sync::Arc; use subprocess::Exec; /// A simple `Codec` implementation that splits up data into lines. @@ -77,19 +76,28 @@ pub(crate) struct ClassifiedPipeline { pub(crate) commands: Vec, } +#[derive(Debug, Eq, PartialEq)] pub(crate) enum ClassifiedCommand { #[allow(unused)] Expr(TokenNode), Internal(InternalCommand), + #[allow(unused)] + Dynamic(hir::Call), External(ExternalCommand), } +#[derive(new, Debug, Eq, PartialEq)] pub(crate) struct InternalCommand { - pub(crate) command: Arc, + pub(crate) name: String, pub(crate) name_tag: Tag, pub(crate) args: hir::Call, } +#[derive(new, Debug, Eq, PartialEq)] +pub(crate) struct DynamicCommand { + pub(crate) args: hir::Call, +} + impl InternalCommand { pub(crate) async fn run( self, @@ -100,15 +108,17 @@ impl InternalCommand { ) -> Result { if log_enabled!(log::Level::Trace) { trace!(target: "nu::run::internal", "->"); - trace!(target: "nu::run::internal", "{}", self.command.name()); + trace!(target: "nu::run::internal", "{}", self.name); trace!(target: "nu::run::internal", "{}", self.args.debug(&source)); } let objects: InputStream = trace_stream!(target: "nu::trace_stream::internal", "input" = input.objects); + let command = context.expect_command(&self.name); + let result = context.run_command( - self.command, + command, self.name_tag.clone(), context.source_map.clone(), self.args, @@ -185,6 +195,7 @@ impl InternalCommand { } } +#[derive(Debug, Eq, PartialEq)] pub(crate) struct ExternalCommand { pub(crate) name: String, @@ -192,6 +203,7 @@ pub(crate) struct ExternalCommand { pub(crate) args: Vec>, } +#[derive(Debug)] pub(crate) enum StreamNext { Last, External, @@ -221,6 +233,8 @@ impl ExternalCommand { process = Exec::cmd(&self.name); + trace!(target: "nu::run::external", "command = {:?}", process); + if arg_string.contains("$it") { let mut first = true; @@ -275,6 +289,8 @@ impl ExternalCommand { process = process.cwd(context.shell_manager.path()); + trace!(target: "nu::run::external", "cwd = {:?}", context.shell_manager.path()); + let mut process = match stream_next { StreamNext::Last => process, StreamNext::External | StreamNext::Internal => { @@ -282,11 +298,18 @@ impl ExternalCommand { } }; + trace!(target: "nu::run::external", "set up stdout pipe"); + if let Some(stdin) = stdin { process = process.stdin(stdin); } - let mut popen = process.popen()?; + trace!(target: "nu::run::external", "set up stdin pipe"); + trace!(target: "nu::run::external", "built process {:?}", process); + + let mut popen = process.popen().unwrap(); + + trace!(target: "nu::run::external", "next = {:?}", stream_next); match stream_next { StreamNext::Last => { diff --git a/src/commands/command.rs b/src/commands/command.rs index 95732abac2..7fb08bcefa 100644 --- a/src/commands/command.rs +++ b/src/commands/command.rs @@ -507,6 +507,15 @@ pub enum Command { PerItem(Arc), } +impl std::fmt::Debug for Command { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Command::WholeStream(command) => write!(f, "WholeStream({})", command.name()), + Command::PerItem(command) => write!(f, "PerItem({})", command.name()), + } + } +} + impl Command { pub fn name(&self) -> &str { match self { diff --git a/src/commands/echo.rs b/src/commands/echo.rs index 21188f54f2..5bfc12efb7 100644 --- a/src/commands/echo.rs +++ b/src/commands/echo.rs @@ -54,11 +54,10 @@ fn run( output.push_str(&s); } _ => { - return Err(ShellError::labeled_error( - "Expect a string from pipeline", - "not a string-compatible value", - i.tag(), - )); + return Err(ShellError::type_error( + "a string-compatible value", + i.tagged_type_name(), + )) } } } diff --git a/src/commands/enter.rs b/src/commands/enter.rs index 2d96fe865c..94688acd56 100644 --- a/src/commands/enter.rs +++ b/src/commands/enter.rs @@ -15,7 +15,7 @@ impl PerItemCommand for Enter { } fn signature(&self) -> registry::Signature { - Signature::build("enter").required("location", SyntaxShape::Block) + Signature::build("enter").required("location", SyntaxShape::Path) } fn usage(&self) -> &str { @@ -33,14 +33,14 @@ impl PerItemCommand for Enter { let raw_args = raw_args.clone(); match call_info.args.expect_nth(0)? { Tagged { - item: Value::Primitive(Primitive::String(location)), + item: Value::Primitive(Primitive::Path(location)), .. } => { - let location = location.to_string(); - let location_clone = location.to_string(); + let location_string = location.display().to_string(); + let location_clone = location_string.clone(); if location.starts_with("help") { - let spec = location.split(":").collect::>(); + let spec = location_string.split(":").collect::>(); let (_, command) = (spec[0], spec[1]); diff --git a/src/commands/fetch.rs b/src/commands/fetch.rs index 652ec77eb5..21ef7fbfd9 100644 --- a/src/commands/fetch.rs +++ b/src/commands/fetch.rs @@ -53,7 +53,7 @@ fn run( }; let path_buf = path.as_path()?; let path_str = path_buf.display().to_string(); - let path_span = path.span(); + let path_span = path.tag.span; let has_raw = call_info.args.has("raw"); let registry = registry.clone(); let raw_args = raw_args.clone(); diff --git a/src/commands/first.rs b/src/commands/first.rs index e39b5155d0..71d05be7e1 100644 --- a/src/commands/first.rs +++ b/src/commands/first.rs @@ -16,7 +16,7 @@ impl WholeStreamCommand for First { } fn signature(&self) -> Signature { - Signature::build("first").required("amount", SyntaxShape::Literal) + Signature::build("first").required("amount", SyntaxShape::Int) } fn usage(&self) -> &str { diff --git a/src/commands/get.rs b/src/commands/get.rs index afa550c72c..4b0916c5d1 100644 --- a/src/commands/get.rs +++ b/src/commands/get.rs @@ -1,14 +1,16 @@ use crate::commands::WholeStreamCommand; +use crate::data::meta::tag_for_tagged_list; use crate::data::Value; use crate::errors::ShellError; use crate::prelude::*; +use log::trace; pub struct Get; #[derive(Deserialize)] pub struct GetArgs { - member: Tagged, - rest: Vec>, + member: ColumnPath, + rest: Vec, } impl WholeStreamCommand for Get { @@ -18,8 +20,8 @@ impl WholeStreamCommand for Get { fn signature(&self) -> Signature { Signature::build("get") - .required("member", SyntaxShape::Member) - .rest(SyntaxShape::Member) + .required("member", SyntaxShape::ColumnPath) + .rest(SyntaxShape::ColumnPath) } fn usage(&self) -> &str { @@ -35,39 +37,34 @@ impl WholeStreamCommand for Get { } } -fn get_member(path: &Tagged, obj: &Tagged) -> Result, ShellError> { +pub type ColumnPath = Vec>; + +pub fn get_column_path( + path: &ColumnPath, + obj: &Tagged, +) -> Result, ShellError> { let mut current = Some(obj); - for p in path.split(".") { + for p in path.iter() { if let Some(obj) = current { - current = match obj.get_data_by_key(p) { + current = match obj.get_data_by_key(&p) { Some(v) => Some(v), None => // Before we give up, see if they gave us a path that matches a field name by itself { - match obj.get_data_by_key(&path.item) { - Some(v) => return Ok(v.clone()), - None => { - let possibilities = obj.data_descriptors(); + let possibilities = obj.data_descriptors(); - let mut possible_matches: Vec<_> = possibilities - .iter() - .map(|x| { - (natural::distance::levenshtein_distance(x, &path.item), x) - }) - .collect(); + let mut possible_matches: Vec<_> = possibilities + .iter() + .map(|x| (natural::distance::levenshtein_distance(x, &p), x)) + .collect(); - possible_matches.sort(); + possible_matches.sort(); - if possible_matches.len() > 0 { - return Err(ShellError::labeled_error( - "Unknown column", - format!("did you mean '{}'?", possible_matches[0].1), - path.tag(), - )); - } - None - } - } + return Err(ShellError::labeled_error( + "Unknown column", + format!("did you mean '{}'?", possible_matches[0].1), + tag_for_tagged_list(path.iter().map(|p| p.tag())), + )); } } } @@ -97,6 +94,8 @@ pub fn get( }: GetArgs, RunnableContext { input, .. }: RunnableContext, ) -> Result { + trace!("get {:?} {:?}", member, fields); + let stream = input .values .map(move |item| { @@ -107,10 +106,10 @@ pub fn get( let fields = vec![&member, &fields] .into_iter() .flatten() - .collect::>>(); + .collect::>(); - for field in &fields { - match get_member(field, &item) { + for column_path in &fields { + match get_column_path(column_path, &item) { Ok(Tagged { item: Value::Table(l), .. diff --git a/src/commands/open.rs b/src/commands/open.rs index 254b0bd7b9..97b0df2744 100644 --- a/src/commands/open.rs +++ b/src/commands/open.rs @@ -54,7 +54,7 @@ fn run( }; let path_buf = path.as_path()?; let path_str = path_buf.display().to_string(); - let path_span = path.span(); + let path_span = path.tag.span; let has_raw = call_info.args.has("raw"); let registry = registry.clone(); let raw_args = raw_args.clone(); diff --git a/src/commands/save.rs b/src/commands/save.rs index 47f1a17e95..44e07da5ed 100644 --- a/src/commands/save.rs +++ b/src/commands/save.rs @@ -143,15 +143,16 @@ fn save( } _ => { yield Err(ShellError::labeled_error( - "Save requires a filepath", + "Save requires a filepath (1)", "needs path", name_tag, )); } }, None => { + eprintln!("{:?} {:?}", anchor, source_map); yield Err(ShellError::labeled_error( - "Save requires a filepath", + "Save requires a filepath (2)", "needs path", name_tag, )); @@ -159,7 +160,7 @@ fn save( } } else { yield Err(ShellError::labeled_error( - "Save requires a filepath", + "Save requires a filepath (3)", "needs path", name_tag, )); diff --git a/src/commands/skip_while.rs b/src/commands/skip_while.rs index 041caf300e..a768ae6133 100644 --- a/src/commands/skip_while.rs +++ b/src/commands/skip_while.rs @@ -1,6 +1,7 @@ use crate::commands::WholeStreamCommand; use crate::errors::ShellError; use crate::prelude::*; +use log::trace; pub struct SkipWhile; @@ -38,7 +39,9 @@ pub fn skip_while( RunnableContext { input, .. }: RunnableContext, ) -> Result { let objects = input.values.skip_while(move |item| { + trace!("ITEM = {:?}", item); let result = condition.invoke(&item); + trace!("RESULT = {:?}", result); let return_value = match result { Ok(ref v) if v.is_true() => true, diff --git a/src/commands/tags.rs b/src/commands/tags.rs index 0cef300b0c..2b710d1b61 100644 --- a/src/commands/tags.rs +++ b/src/commands/tags.rs @@ -38,8 +38,8 @@ fn tags(args: CommandArgs, _registry: &CommandRegistry) -> Result Arc { + self.get_command(name).unwrap() + } + pub(crate) fn has(&self, name: &str) -> bool { let registry = self.registry.lock().unwrap(); registry.contains_key(name) } - fn insert(&mut self, name: impl Into, command: Arc) { + pub(crate) fn insert(&mut self, name: impl Into, command: Arc) { let mut registry = self.registry.lock().unwrap(); registry.insert(name.into(), command); } @@ -83,6 +87,14 @@ impl Context { &self.registry } + pub(crate) fn expand_context<'context>( + &'context self, + source: &'context Text, + tag: Tag, + ) -> ExpandContext<'context> { + ExpandContext::new(&self.registry, tag, source, self.shell_manager.homedir()) + } + pub(crate) fn basic() -> Result> { let registry = CommandRegistry::new(); Ok(Context { @@ -109,12 +121,12 @@ impl Context { self.source_map.insert(uuid, anchor_location); } - pub(crate) fn has_command(&self, name: &str) -> bool { - self.registry.has(name) + pub(crate) fn get_command(&self, name: &str) -> Option> { + self.registry.get_command(name) } - pub(crate) fn get_command(&self, name: &str) -> Arc { - self.registry.get_command(name).unwrap() + pub(crate) fn expect_command(&self, name: &str) -> Arc { + self.registry.expect_command(name) } pub(crate) fn run_command<'a>( diff --git a/src/data/base.rs b/src/data/base.rs index 04465181a3..176560137f 100644 --- a/src/data/base.rs +++ b/src/data/base.rs @@ -8,6 +8,7 @@ use crate::Text; use chrono::{DateTime, Utc}; use chrono_humanize::Humanize; use derive_new::new; +use log::trace; use serde::{Deserialize, Serialize}; use std::fmt; use std::path::PathBuf; @@ -217,6 +218,14 @@ impl Block { let mut last = None; + trace!( + "EXPRS = {:?}", + self.expressions + .iter() + .map(|e| format!("{}", e)) + .collect::>() + ); + for expr in self.expressions.iter() { last = Some(evaluate_baseline_expr( &expr, @@ -394,13 +403,34 @@ impl Tagged { pub(crate) fn debug(&self) -> ValueDebug<'_> { ValueDebug { value: self } } + + pub fn as_column_path(&self) -> Result>>, ShellError> { + let mut out: Vec> = vec![]; + + match &self.item { + Value::Table(table) => { + for item in table { + out.push(item.as_string()?.tagged(item.tag)); + } + } + + other => { + return Err(ShellError::type_error( + "column name", + other.type_name().tagged(self.tag), + )) + } + } + + Ok(out.tagged(self.tag)) + } } impl Value { pub(crate) fn type_name(&self) -> String { match self { Value::Primitive(p) => p.type_name(), - Value::Row(_) => format!("object"), + Value::Row(_) => format!("row"), Value::Table(_) => format!("list"), Value::Block(_) => format!("block"), } @@ -443,6 +473,22 @@ impl Value { } } + pub fn get_data_by_column_path( + &self, + tag: Tag, + path: &Vec>, + ) -> Option> { + let mut current = self; + for p in path { + match current.get_data_by_key(p) { + Some(v) => current = v, + None => return None, + } + } + + Some(Tagged::from_item(current, tag)) + } + pub fn get_data_by_path(&self, tag: Tag, path: &str) -> Option> { let mut current = self; for p in path.split(".") { @@ -508,6 +554,58 @@ impl Value { None } + pub fn insert_data_at_column_path( + &self, + tag: Tag, + split_path: &Vec>, + new_value: Value, + ) -> Option> { + let mut new_obj = self.clone(); + + if let Value::Row(ref mut o) = new_obj { + let mut current = o; + + if split_path.len() == 1 { + // Special case for inserting at the top level + current.entries.insert( + split_path[0].item.clone(), + Tagged::from_item(new_value, tag), + ); + return Some(Tagged::from_item(new_obj, tag)); + } + + for idx in 0..split_path.len() { + match current.entries.get_mut(&split_path[idx].item) { + Some(next) => { + if idx == (split_path.len() - 2) { + match &mut next.item { + Value::Row(o) => { + o.entries.insert( + split_path[idx + 1].to_string(), + Tagged::from_item(new_value, tag), + ); + } + _ => {} + } + + return Some(Tagged::from_item(new_obj, tag)); + } else { + match next.item { + Value::Row(ref mut o) => { + current = o; + } + _ => return None, + } + } + } + _ => return None, + } + } + } + + None + } + pub fn replace_data_at_path( &self, tag: Tag, @@ -543,6 +641,39 @@ impl Value { None } + pub fn replace_data_at_column_path( + &self, + tag: Tag, + split_path: &Vec>, + replaced_value: Value, + ) -> Option> { + let mut new_obj = self.clone(); + + if let Value::Row(ref mut o) = new_obj { + let mut current = o; + for idx in 0..split_path.len() { + match current.entries.get_mut(&split_path[idx].item) { + Some(next) => { + if idx == (split_path.len() - 1) { + *next = Tagged::from_item(replaced_value, tag); + return Some(Tagged::from_item(new_obj, tag)); + } else { + match next.item { + Value::Row(ref mut o) => { + current = o; + } + _ => return None, + } + } + } + _ => return None, + } + } + } + + None + } + pub fn get_data(&self, desc: &String) -> MaybeOwned<'_, Value> { match self { p @ Value::Primitive(_) => MaybeOwned::Borrowed(p), diff --git a/src/data/meta.rs b/src/data/meta.rs index 0a56198e6c..b66b009cc2 100644 --- a/src/data/meta.rs +++ b/src/data/meta.rs @@ -1,4 +1,5 @@ use crate::context::{AnchorLocation, SourceMap}; +use crate::parser::parse::parser::TracableContext; use crate::prelude::*; use crate::Text; use derive_new::new; @@ -119,10 +120,7 @@ impl From<&Tag> for Tag { impl From> for Span { fn from(input: nom_locate::LocatedSpanEx<&str, Uuid>) -> Span { - Span { - start: input.offset, - end: input.offset + input.fragment.len(), - } + Span::new(input.offset, input.offset + input.fragment.len()) } } @@ -147,10 +145,7 @@ impl impl From<(usize, usize)> for Span { fn from(input: (usize, usize)) -> Span { - Span { - start: input.0, - end: input.1, - } + Span::new(input.0, input.1) } } @@ -164,7 +159,7 @@ impl From<&std::ops::Range> for Span { } #[derive( - Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash, Getters, + Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash, Getters, new, )] pub struct Tag { pub anchor: Uuid, @@ -189,11 +184,20 @@ impl From<&Span> for Tag { } } +impl From<(usize, usize, TracableContext)> for Tag { + fn from((start, end, context): (usize, usize, TracableContext)) -> Self { + Tag { + anchor: context.origin, + span: Span::new(start, end), + } + } +} + impl From<(usize, usize, Uuid)> for Tag { fn from((start, end, anchor): (usize, usize, Uuid)) -> Self { Tag { anchor, - span: Span { start, end }, + span: Span::new(start, end), } } } @@ -201,24 +205,17 @@ impl From<(usize, usize, Uuid)> for Tag { impl From<(usize, usize, Option)> for Tag { fn from((start, end, anchor): (usize, usize, Option)) -> Self { Tag { - anchor: if let Some(uuid) = anchor { - uuid - } else { - uuid::Uuid::nil() - }, - span: Span { start, end }, + anchor: anchor.unwrap_or(uuid::Uuid::nil()), + span: Span::new(start, end), } } } -impl From> for Tag { - fn from(input: nom_locate::LocatedSpanEx<&str, Uuid>) -> Tag { +impl From> for Tag { + fn from(input: nom_locate::LocatedSpanEx<&str, TracableContext>) -> Tag { Tag { - anchor: input.extra, - span: Span { - start: input.offset, - end: input.offset + input.fragment.len(), - }, + anchor: input.extra.origin, + span: Span::new(input.offset, input.offset + input.fragment.len()), } } } @@ -265,10 +262,7 @@ impl Tag { ); Tag { - span: Span { - start: self.span.start, - end: other.span.end, - }, + span: Span::new(self.span.start, other.span.end), anchor: self.anchor, } } @@ -276,18 +270,46 @@ impl Tag { pub fn slice<'a>(&self, source: &'a str) -> &'a str { self.span.slice(source) } + + pub fn string<'a>(&self, source: &'a str) -> String { + self.span.slice(source).to_string() + } + + pub fn tagged_slice<'a>(&self, source: &'a str) -> Tagged<&'a str> { + self.span.slice(source).tagged(self) + } + + pub fn tagged_string<'a>(&self, source: &'a str) -> Tagged { + self.span.slice(source).to_string().tagged(self) + } +} + +pub fn tag_for_tagged_list(mut iter: impl Iterator) -> Tag { + let first = iter.next(); + + let first = match first { + None => return Tag::unknown(), + Some(first) => first, + }; + + let last = iter.last(); + + match last { + None => first, + Some(last) => first.until(last), + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash)] pub struct Span { - pub(crate) start: usize, - pub(crate) end: usize, + start: usize, + end: usize, } impl From> for Span { fn from(input: Option) -> Span { match input { - None => Span { start: 0, end: 0 }, + None => Span::new(0, 0), Some(span) => span, } } @@ -295,7 +317,18 @@ impl From> for Span { impl Span { pub fn unknown() -> Span { - Span { start: 0, end: 0 } + Span::new(0, 0) + } + + pub fn new(start: usize, end: usize) -> Span { + assert!( + end >= start, + "Can't create a Span whose end < start, start={}, end={}", + start, + end + ); + + Span { start, end } } /* @@ -308,6 +341,14 @@ impl Span { } */ + pub fn start(&self) -> usize { + self.start + } + + pub fn end(&self) -> usize { + self.end + } + pub fn is_unknown(&self) -> bool { self.start == 0 && self.end == 0 } @@ -319,17 +360,11 @@ impl Span { impl language_reporting::ReportingSpan for Span { fn with_start(&self, start: usize) -> Self { - Span { - start, - end: self.end, - } + Span::new(start, self.end) } fn with_end(&self, end: usize) -> Self { - Span { - start: self.start, - end, - } + Span::new(self.start, end) } fn start(&self) -> usize { @@ -344,20 +379,14 @@ impl language_reporting::ReportingSpan for Span { impl language_reporting::ReportingSpan for Tag { fn with_start(&self, start: usize) -> Self { Tag { - span: Span { - start, - end: self.span.end, - }, + span: Span::new(start, self.span.end), anchor: self.anchor, } } fn with_end(&self, end: usize) -> Self { Tag { - span: Span { - start: self.span.start, - end, - }, + span: Span::new(self.span.start, end), anchor: self.anchor, } } diff --git a/src/errors.rs b/src/errors.rs index 7e9c14b239..a070f6f54e 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,5 +1,6 @@ use crate::prelude::*; +use crate::parser::parse::parser::TracableContext; use ansi_term::Color; use derive_new::new; use language_reporting::{Diagnostic, Label, Severity}; @@ -62,6 +63,14 @@ impl ShellError { .start() } + pub(crate) fn unexpected_eof(expected: impl Into, tag: Tag) -> ShellError { + ProximateShellError::UnexpectedEof { + expected: expected.into(), + tag, + } + .start() + } + pub(crate) fn range_error( expected: impl Into, actual: &Tagged, @@ -82,6 +91,7 @@ impl ShellError { .start() } + #[allow(unused)] pub(crate) fn invalid_command(problem: impl Into) -> ShellError { ProximateShellError::InvalidCommand { command: problem.into(), @@ -133,7 +143,7 @@ impl ShellError { pub(crate) fn parse_error( error: nom::Err<( - nom_locate::LocatedSpanEx<&str, uuid::Uuid>, + nom_locate::LocatedSpanEx<&str, TracableContext>, nom::error::ErrorKind, )>, ) -> ShellError { @@ -235,7 +245,6 @@ impl ShellError { Label::new_primary(tag) .with_message(format!("Expected {}, found {}", expected, actual)), ), - ProximateShellError::TypeError { expected, actual: @@ -246,6 +255,11 @@ impl ShellError { } => Diagnostic::new(Severity::Error, "Type Error") .with_label(Label::new_primary(tag).with_message(expected)), + ProximateShellError::UnexpectedEof { + expected, tag + } => Diagnostic::new(Severity::Error, format!("Unexpected end of input")) + .with_label(Label::new_primary(tag).with_message(format!("Expected {}", expected))), + ProximateShellError::RangeError { kind, operation, @@ -267,10 +281,10 @@ impl ShellError { problem: Tagged { tag, - .. + item }, } => Diagnostic::new(Severity::Error, "Syntax Error") - .with_label(Label::new_primary(tag).with_message("Unexpected external command")), + .with_label(Label::new_primary(tag).with_message(item)), ProximateShellError::MissingProperty { subpath, expr } => { let subpath = subpath.into_label(); @@ -340,6 +354,10 @@ impl ShellError { pub(crate) fn unexpected(title: impl Into) -> ShellError { ShellError::string(&format!("Unexpected: {}", title.into())) } + + pub(crate) fn unreachable(title: impl Into) -> ShellError { + ShellError::string(&format!("BUG: Unreachable: {}", title.into())) + } } #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)] @@ -387,6 +405,10 @@ pub enum ProximateShellError { SyntaxError { problem: Tagged, }, + UnexpectedEof { + expected: String, + tag: Tag, + }, InvalidCommand { command: Tag, }, @@ -473,6 +495,7 @@ impl std::fmt::Display for ShellError { ProximateShellError::MissingValue { .. } => write!(f, "MissingValue"), ProximateShellError::InvalidCommand { .. } => write!(f, "InvalidCommand"), ProximateShellError::TypeError { .. } => write!(f, "TypeError"), + ProximateShellError::UnexpectedEof { .. } => write!(f, "UnexpectedEof"), ProximateShellError::RangeError { .. } => write!(f, "RangeError"), ProximateShellError::SyntaxError { .. } => write!(f, "SyntaxError"), ProximateShellError::MissingProperty { .. } => write!(f, "MissingProperty"), diff --git a/src/evaluate/evaluator.rs b/src/evaluate/evaluator.rs index a111d3964d..248d2a0816 100644 --- a/src/evaluate/evaluator.rs +++ b/src/evaluate/evaluator.rs @@ -7,6 +7,8 @@ use crate::parser::{ use crate::prelude::*; use derive_new::new; use indexmap::IndexMap; +use log::trace; +use std::fmt; #[derive(new)] pub struct Scope { @@ -15,6 +17,15 @@ pub struct Scope { vars: IndexMap>, } +impl fmt::Display for Scope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_map() + .entry(&"$it", &format!("{:?}", self.it.item)) + .entries(self.vars.iter().map(|(k, v)| (k, &v.item))) + .finish() + } +} + impl Scope { pub(crate) fn empty() -> Scope { Scope { @@ -48,12 +59,15 @@ pub(crate) fn evaluate_baseline_expr( RawExpression::Synthetic(hir::Synthetic::String(s)) => { Ok(Value::string(s).tagged_unknown()) } - RawExpression::Variable(var) => evaluate_reference(var, scope, source), + RawExpression::Variable(var) => evaluate_reference(var, scope, source, expr.tag()), + RawExpression::Command(_) => evaluate_command(expr.tag(), scope, source), RawExpression::ExternalCommand(external) => evaluate_external(external, scope, source), RawExpression::Binary(binary) => { let left = evaluate_baseline_expr(binary.left(), registry, scope, source)?; let right = evaluate_baseline_expr(binary.right(), registry, scope, source)?; + trace!("left={:?} right={:?}", left.item, right.item); + match left.compare(binary.op(), &*right) { Ok(result) => Ok(Value::boolean(result).tagged(expr.tag())), Err((left_type, right_type)) => Err(ShellError::coerce_error( @@ -130,14 +144,16 @@ fn evaluate_reference( name: &hir::Variable, scope: &Scope, source: &Text, + tag: Tag, ) -> Result, ShellError> { + trace!("Evaluating {} with Scope {}", name, scope); match name { - hir::Variable::It(tag) => Ok(scope.it.item.clone().tagged(*tag)), - hir::Variable::Other(tag) => Ok(scope + hir::Variable::It(_) => Ok(scope.it.item.clone().tagged(tag)), + hir::Variable::Other(inner) => Ok(scope .vars - .get(tag.slice(source)) + .get(inner.slice(source)) .map(|v| v.clone()) - .unwrap_or_else(|| Value::nothing().tagged(*tag))), + .unwrap_or_else(|| Value::nothing().tagged(tag))), } } @@ -150,3 +166,7 @@ fn evaluate_external( "Unexpected external command".tagged(*external.name()), )) } + +fn evaluate_command(tag: Tag, _scope: &Scope, _source: &Text) -> Result, ShellError> { + Err(ShellError::syntax_error("Unexpected command".tagged(tag))) +} diff --git a/src/lib.rs b/src/lib.rs index e8e09aacdd..b955f426e9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,7 +31,7 @@ pub use cli::cli; pub use data::base::{Primitive, Value}; pub use data::config::{config_path, APP_INFO}; pub use data::dict::{Dictionary, TaggedDictBuilder}; -pub use data::meta::{Tag, Tagged, TaggedItem}; +pub use data::meta::{Span, Tag, Tagged, TaggedItem}; pub use errors::{CoerceInto, ShellError}; pub use num_traits::cast::ToPrimitive; pub use parser::parse::text::Text; diff --git a/src/parser.rs b/src/parser.rs index 138125769b..5fcfaaa27e 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -7,7 +7,7 @@ pub(crate) mod registry; use crate::errors::ShellError; pub(crate) use deserializer::ConfigDeserializer; -pub(crate) use hir::baseline_parse_tokens::baseline_parse_tokens; +pub(crate) use hir::TokensIterator; pub(crate) use parse::call_node::CallNode; pub(crate) use parse::files::Files; pub(crate) use parse::flag::Flag; @@ -15,10 +15,10 @@ pub(crate) use parse::operator::Operator; pub(crate) use parse::parser::{nom_input, pipeline}; pub(crate) use parse::pipeline::{Pipeline, PipelineElement}; pub(crate) use parse::text::Text; -pub(crate) use parse::token_tree::{DelimitedNode, Delimiter, PathNode, TokenNode}; +pub(crate) use parse::token_tree::{DelimitedNode, Delimiter, TokenNode}; pub(crate) use parse::tokens::{RawToken, Token}; pub(crate) use parse::unit::Unit; -pub(crate) use parse_command::parse_command; +pub(crate) use parse_command::parse_command_tail; pub(crate) use registry::CommandRegistry; pub fn parse(input: &str, anchor: uuid::Uuid) -> Result { diff --git a/src/parser/deserializer.rs b/src/parser/deserializer.rs index f9b9146e50..43409fc4df 100644 --- a/src/parser/deserializer.rs +++ b/src/parser/deserializer.rs @@ -310,9 +310,10 @@ impl<'de, 'a> de::Deserializer<'de> for &'a mut ConfigDeserializer<'de> { return Ok(r); } trace!( - "deserializing struct {:?} {:?} (stack={:?})", + "deserializing struct {:?} {:?} (saw_root={} stack={:?})", name, fields, + self.saw_root, self.stack ); @@ -326,6 +327,12 @@ impl<'de, 'a> de::Deserializer<'de> for &'a mut ConfigDeserializer<'de> { let type_name = std::any::type_name::(); let tagged_val_name = std::any::type_name::>(); + trace!( + "type_name={} tagged_val_name={}", + type_name, + tagged_val_name + ); + if type_name == tagged_val_name { return visit::, _>(value.val, name, fields, visitor); } diff --git a/src/parser/hir.rs b/src/parser/hir.rs index 96eb7272a6..4fd0a71b3d 100644 --- a/src/parser/hir.rs +++ b/src/parser/hir.rs @@ -1,11 +1,13 @@ pub(crate) mod baseline_parse; -pub(crate) mod baseline_parse_tokens; pub(crate) mod binary; +pub(crate) mod expand_external_tokens; pub(crate) mod external_command; pub(crate) mod named; pub(crate) mod path; +pub(crate) mod syntax_shape; +pub(crate) mod tokens_iterator; -use crate::parser::{registry, Unit}; +use crate::parser::{registry, Operator, Unit}; use crate::prelude::*; use derive_new::new; use getset::Getters; @@ -14,27 +16,18 @@ use std::fmt; use std::path::PathBuf; use crate::evaluate::Scope; +use crate::parser::parse::tokens::RawNumber; +use crate::traits::ToDebug; -pub(crate) use self::baseline_parse::{ - baseline_parse_single_token, baseline_parse_token_as_number, baseline_parse_token_as_path, - baseline_parse_token_as_pattern, baseline_parse_token_as_string, -}; -pub(crate) use self::baseline_parse_tokens::{baseline_parse_next_expr, TokensIterator}; pub(crate) use self::binary::Binary; pub(crate) use self::external_command::ExternalCommand; pub(crate) use self::named::NamedArguments; pub(crate) use self::path::Path; +pub(crate) use self::syntax_shape::ExpandContext; +pub(crate) use self::tokens_iterator::debug::debug_tokens; +pub(crate) use self::tokens_iterator::TokensIterator; -pub use self::baseline_parse_tokens::SyntaxShape; - -pub fn path(head: impl Into, tail: Vec>>) -> Path { - Path::new( - head.into(), - tail.into_iter() - .map(|item| item.map(|string| string.into())) - .collect(), - ) -} +pub use self::syntax_shape::SyntaxShape; #[derive(Debug, Clone, Eq, PartialEq, Getters, Serialize, Deserialize, new)] pub struct Call { @@ -93,6 +86,7 @@ pub enum RawExpression { FilePath(PathBuf), ExternalCommand(ExternalCommand), + Command(Tag), Boolean(bool), } @@ -115,13 +109,14 @@ impl RawExpression { match self { RawExpression::Literal(literal) => literal.type_name(), RawExpression::Synthetic(synthetic) => synthetic.type_name(), - RawExpression::ExternalWord => "externalword", - RawExpression::FilePath(..) => "filepath", + RawExpression::Command(..) => "command", + RawExpression::ExternalWord => "external word", + RawExpression::FilePath(..) => "file path", RawExpression::Variable(..) => "variable", RawExpression::List(..) => "list", RawExpression::Binary(..) => "binary", RawExpression::Block(..) => "block", - RawExpression::Path(..) => "path", + RawExpression::Path(..) => "variable path", RawExpression::Boolean(..) => "boolean", RawExpression::ExternalCommand(..) => "external", } @@ -130,6 +125,39 @@ impl RawExpression { pub type Expression = Tagged; +impl std::fmt::Display for Expression { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let span = self.tag.span; + + match &self.item { + RawExpression::Literal(literal) => write!(f, "{}", literal.tagged(self.tag)), + RawExpression::Synthetic(Synthetic::String(s)) => write!(f, "{}", s), + RawExpression::Command(_) => write!(f, "Command{{ {}..{} }}", span.start(), span.end()), + RawExpression::ExternalWord => { + write!(f, "ExternalWord{{ {}..{} }}", span.start(), span.end()) + } + RawExpression::FilePath(file) => write!(f, "Path{{ {} }}", file.display()), + RawExpression::Variable(variable) => write!(f, "{}", variable), + RawExpression::List(list) => f + .debug_list() + .entries(list.iter().map(|e| format!("{}", e))) + .finish(), + RawExpression::Binary(binary) => write!(f, "{}", binary), + RawExpression::Block(items) => { + write!(f, "Block")?; + f.debug_set() + .entries(items.iter().map(|i| format!("{}", i))) + .finish() + } + RawExpression::Path(path) => write!(f, "{}", path), + RawExpression::Boolean(b) => write!(f, "${}", b), + RawExpression::ExternalCommand(..) => { + write!(f, "ExternalComment{{ {}..{} }}", span.start(), span.end()) + } + } + } +} + impl Expression { pub(crate) fn number(i: impl Into, tag: impl Into) -> Expression { RawExpression::Literal(Literal::Number(i.into())).tagged(tag.into()) @@ -151,10 +179,50 @@ impl Expression { RawExpression::Literal(Literal::String(inner.into())).tagged(outer.into()) } + pub(crate) fn path( + head: Expression, + tail: Vec>>, + tag: impl Into, + ) -> Expression { + let tail = tail.into_iter().map(|t| t.map(|s| s.into())).collect(); + RawExpression::Path(Box::new(Path::new(head, tail))).tagged(tag.into()) + } + + pub(crate) fn dot_member(head: Expression, next: Tagged>) -> Expression { + let Tagged { item, tag } = head; + let new_tag = head.tag.until(next.tag); + + match item { + RawExpression::Path(path) => { + let (head, mut tail) = path.parts(); + + tail.push(next.map(|i| i.into())); + Expression::path(head, tail, new_tag) + } + + other => Expression::path(other.tagged(tag), vec![next], new_tag), + } + } + + pub(crate) fn infix( + left: Expression, + op: Tagged>, + right: Expression, + ) -> Expression { + let new_tag = left.tag.until(right.tag); + + RawExpression::Binary(Box::new(Binary::new(left, op.map(|o| o.into()), right))) + .tagged(new_tag) + } + pub(crate) fn file_path(path: impl Into, outer: impl Into) -> Expression { RawExpression::FilePath(path.into()).tagged(outer) } + pub(crate) fn list(list: Vec, tag: impl Into) -> Expression { + RawExpression::List(list).tagged(tag) + } + pub(crate) fn bare(tag: impl Into) -> Expression { RawExpression::Literal(Literal::Bare).tagged(tag) } @@ -182,6 +250,7 @@ impl ToDebug for Expression { RawExpression::Literal(l) => l.tagged(self.tag()).fmt_debug(f, source), RawExpression::FilePath(p) => write!(f, "{}", p.display()), RawExpression::ExternalWord => write!(f, "{}", self.tag().slice(source)), + RawExpression::Command(tag) => write!(f, "{}", tag.slice(source)), RawExpression::Synthetic(Synthetic::String(s)) => write!(f, "{:?}", s), RawExpression::Variable(Variable::It(_)) => write!(f, "$it"), RawExpression::Variable(Variable::Other(s)) => write!(f, "${}", s.slice(source)), @@ -232,6 +301,26 @@ pub enum Literal { Bare, } +impl std::fmt::Display for Tagged { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", Tagged::new(self.tag, &self.item)) + } +} + +impl std::fmt::Display for Tagged<&Literal> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let span = self.tag.span; + + match &self.item { + Literal::Number(number) => write!(f, "{}", number), + Literal::Size(number, unit) => write!(f, "{}{}", number, unit.as_str()), + Literal::String(_) => write!(f, "String{{ {}..{} }}", span.start(), span.end()), + Literal::GlobPattern => write!(f, "Glob{{ {}..{} }}", span.start(), span.end()), + Literal::Bare => write!(f, "Bare{{ {}..{} }}", span.start(), span.end()), + } + } +} + impl ToDebug for Tagged<&Literal> { fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { match self.item() { @@ -261,3 +350,12 @@ pub enum Variable { It(Tag), Other(Tag), } + +impl std::fmt::Display for Variable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Variable::It(_) => write!(f, "$it"), + Variable::Other(tag) => write!(f, "${{ {}..{} }}", tag.span.start(), tag.span.end()), + } + } +} diff --git a/src/parser/hir/baseline_parse.rs b/src/parser/hir/baseline_parse.rs index 267494f27c..87c2771955 100644 --- a/src/parser/hir/baseline_parse.rs +++ b/src/parser/hir/baseline_parse.rs @@ -1,140 +1,2 @@ -use crate::context::Context; -use crate::errors::ShellError; -use crate::parser::{hir, RawToken, Token}; -use crate::TaggedItem; -use crate::Text; -use std::path::PathBuf; - -pub fn baseline_parse_single_token( - token: &Token, - source: &Text, -) -> Result { - Ok(match *token.item() { - RawToken::Number(number) => hir::Expression::number(number.to_number(source), token.tag()), - RawToken::Size(int, unit) => { - hir::Expression::size(int.to_number(source), unit, token.tag()) - } - RawToken::String(tag) => hir::Expression::string(tag, token.tag()), - RawToken::Variable(tag) if tag.slice(source) == "it" => { - hir::Expression::it_variable(tag, token.tag()) - } - RawToken::Variable(tag) => hir::Expression::variable(tag, token.tag()), - RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token.tag()), - RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.tag())), - RawToken::GlobPattern => hir::Expression::pattern(token.tag()), - RawToken::Bare => hir::Expression::bare(token.tag()), - }) -} - -pub fn baseline_parse_token_as_number( - token: &Token, - source: &Text, -) -> Result { - Ok(match *token.item() { - RawToken::Variable(tag) if tag.slice(source) == "it" => { - hir::Expression::it_variable(tag, token.tag()) - } - RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token.tag()), - RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.tag())), - RawToken::Variable(tag) => hir::Expression::variable(tag, token.tag()), - RawToken::Number(number) => hir::Expression::number(number.to_number(source), token.tag()), - RawToken::Size(number, unit) => { - hir::Expression::size(number.to_number(source), unit, token.tag()) - } - RawToken::Bare => hir::Expression::bare(token.tag()), - RawToken::GlobPattern => { - return Err(ShellError::type_error( - "Number", - "glob pattern".to_string().tagged(token.tag()), - )) - } - RawToken::String(tag) => hir::Expression::string(tag, token.tag()), - }) -} - -pub fn baseline_parse_token_as_string( - token: &Token, - source: &Text, -) -> Result { - Ok(match *token.item() { - RawToken::Variable(tag) if tag.slice(source) == "it" => { - hir::Expression::it_variable(tag, token.tag()) - } - RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token.tag()), - RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.tag())), - RawToken::Variable(tag) => hir::Expression::variable(tag, token.tag()), - RawToken::Number(_) => hir::Expression::bare(token.tag()), - RawToken::Size(_, _) => hir::Expression::bare(token.tag()), - RawToken::Bare => hir::Expression::bare(token.tag()), - RawToken::GlobPattern => { - return Err(ShellError::type_error( - "String", - "glob pattern".tagged(token.tag()), - )) - } - RawToken::String(tag) => hir::Expression::string(tag, token.tag()), - }) -} - -pub fn baseline_parse_token_as_path( - token: &Token, - context: &Context, - source: &Text, -) -> Result { - Ok(match *token.item() { - RawToken::Variable(tag) if tag.slice(source) == "it" => { - hir::Expression::it_variable(tag, token.tag()) - } - RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token.tag()), - RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.tag())), - RawToken::Variable(tag) => hir::Expression::variable(tag, token.tag()), - RawToken::Number(_) => hir::Expression::bare(token.tag()), - RawToken::Size(_, _) => hir::Expression::bare(token.tag()), - RawToken::Bare => { - hir::Expression::file_path(expand_path(token.tag().slice(source), context), token.tag()) - } - RawToken::GlobPattern => { - return Err(ShellError::type_error( - "Path", - "glob pattern".tagged(token.tag()), - )) - } - RawToken::String(tag) => { - hir::Expression::file_path(expand_path(tag.slice(source), context), token.tag()) - } - }) -} - -pub fn baseline_parse_token_as_pattern( - token: &Token, - context: &Context, - source: &Text, -) -> Result { - Ok(match *token.item() { - RawToken::Variable(tag) if tag.slice(source) == "it" => { - hir::Expression::it_variable(tag, token.tag()) - } - RawToken::ExternalCommand(_) => { - return Err(ShellError::syntax_error( - "Invalid external command".to_string().tagged(token.tag()), - )) - } - RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.tag())), - RawToken::Variable(tag) => hir::Expression::variable(tag, token.tag()), - RawToken::Number(_) => hir::Expression::bare(token.tag()), - RawToken::Size(_, _) => hir::Expression::bare(token.tag()), - RawToken::GlobPattern => hir::Expression::pattern(token.tag()), - RawToken::Bare => { - hir::Expression::file_path(expand_path(token.tag().slice(source), context), token.tag()) - } - RawToken::String(tag) => { - hir::Expression::file_path(expand_path(tag.slice(source), context), token.tag()) - } - }) -} - -pub fn expand_path(string: &str, context: &Context) -> PathBuf { - let expanded = shellexpand::tilde_with_context(string, || context.shell_manager.homedir()); - - PathBuf::from(expanded.as_ref()) -} +#[cfg(test)] +mod tests; diff --git a/src/parser/hir/baseline_parse/tests.rs b/src/parser/hir/baseline_parse/tests.rs new file mode 100644 index 0000000000..badb177513 --- /dev/null +++ b/src/parser/hir/baseline_parse/tests.rs @@ -0,0 +1,144 @@ +use crate::commands::classified::InternalCommand; +use crate::commands::ClassifiedCommand; +use crate::env::host::BasicHost; +use crate::parser::hir; +use crate::parser::hir::syntax_shape::*; +use crate::parser::hir::TokensIterator; +use crate::parser::parse::token_tree_builder::{CurriedToken, TokenTreeBuilder as b}; +use crate::parser::TokenNode; +use crate::{Span, Tag, Tagged, TaggedItem, Text}; +use pretty_assertions::assert_eq; +use std::fmt::Debug; +use uuid::Uuid; + +#[test] +fn test_parse_string() { + parse_tokens(StringShape, vec![b::string("hello")], |tokens| { + hir::Expression::string(inner_string_tag(tokens[0].tag()), tokens[0].tag()) + }); +} + +#[test] +fn test_parse_path() { + parse_tokens( + VariablePathShape, + vec![b::var("it"), b::op("."), b::bare("cpu")], + |tokens| { + let (outer_var, inner_var) = tokens[0].expect_var(); + let bare = tokens[2].expect_bare(); + hir::Expression::path( + hir::Expression::it_variable(inner_var, outer_var), + vec!["cpu".tagged(bare)], + outer_var.until(bare), + ) + }, + ); + + parse_tokens( + VariablePathShape, + vec![ + b::var("cpu"), + b::op("."), + b::bare("amount"), + b::op("."), + b::string("max ghz"), + ], + |tokens| { + let (outer_var, inner_var) = tokens[0].expect_var(); + let amount = tokens[2].expect_bare(); + let (outer_max_ghz, _) = tokens[4].expect_string(); + + hir::Expression::path( + hir::Expression::variable(inner_var, outer_var), + vec!["amount".tagged(amount), "max ghz".tagged(outer_max_ghz)], + outer_var.until(outer_max_ghz), + ) + }, + ); +} + +#[test] +fn test_parse_command() { + parse_tokens( + ClassifiedCommandShape, + vec![b::bare("ls"), b::sp(), b::pattern("*.txt")], + |tokens| { + let bare = tokens[0].expect_bare(); + let pat = tokens[2].tag(); + + ClassifiedCommand::Internal(InternalCommand::new( + "ls".to_string(), + bare, + hir::Call { + head: Box::new(hir::RawExpression::Command(bare).tagged(bare)), + positional: Some(vec![hir::Expression::pattern(pat)]), + named: None, + }, + )) + // hir::Expression::path( + // hir::Expression::variable(inner_var, outer_var), + // vec!["cpu".tagged(bare)], + // outer_var.until(bare), + // ) + }, + ); + + parse_tokens( + VariablePathShape, + vec![ + b::var("cpu"), + b::op("."), + b::bare("amount"), + b::op("."), + b::string("max ghz"), + ], + |tokens| { + let (outer_var, inner_var) = tokens[0].expect_var(); + let amount = tokens[2].expect_bare(); + let (outer_max_ghz, _) = tokens[4].expect_string(); + + hir::Expression::path( + hir::Expression::variable(inner_var, outer_var), + vec!["amount".tagged(amount), "max ghz".tagged(outer_max_ghz)], + outer_var.until(outer_max_ghz), + ) + }, + ); +} + +fn parse_tokens( + shape: impl ExpandSyntax, + tokens: Vec, + expected: impl FnOnce(Tagged<&[TokenNode]>) -> T, +) { + let tokens = b::token_list(tokens); + let (tokens, source) = b::build(test_origin(), tokens); + + ExpandContext::with_empty(&Text::from(source), |context| { + let tokens = tokens.expect_list(); + let mut iterator = TokensIterator::all(tokens.item, *context.tag()); + + let expr = expand_syntax(&shape, &mut iterator, &context); + + let expr = match expr { + Ok(expr) => expr, + Err(err) => { + crate::cli::print_err(err, &BasicHost, context.source().clone()); + panic!("Parse failed"); + } + }; + + assert_eq!(expr, expected(tokens)); + }) +} + +fn test_origin() -> Uuid { + Uuid::nil() +} + +fn inner_string_tag(tag: Tag) -> Tag { + Tag { + span: Span::new(tag.span.start() + 1, tag.span.end() - 1), + anchor: tag.anchor, + } +} diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs deleted file mode 100644 index 8413bd07e1..0000000000 --- a/src/parser/hir/baseline_parse_tokens.rs +++ /dev/null @@ -1,459 +0,0 @@ -use crate::context::Context; -use crate::errors::ShellError; -use crate::parser::{ - hir, - hir::{ - baseline_parse_single_token, baseline_parse_token_as_number, baseline_parse_token_as_path, - baseline_parse_token_as_pattern, baseline_parse_token_as_string, - }, - DelimitedNode, Delimiter, PathNode, RawToken, TokenNode, -}; -use crate::{Tag, Tagged, TaggedItem, Text}; -use derive_new::new; -use log::trace; -use serde::{Deserialize, Serialize}; - -pub fn baseline_parse_tokens( - token_nodes: &mut TokensIterator<'_>, - context: &Context, - source: &Text, - syntax_type: SyntaxShape, -) -> Result, ShellError> { - let mut exprs: Vec = vec![]; - - loop { - if token_nodes.at_end() { - break; - } - - let expr = baseline_parse_next_expr(token_nodes, context, source, syntax_type)?; - exprs.push(expr); - } - - Ok(exprs) -} - -#[derive(Debug, Copy, Clone, Serialize, Deserialize)] -pub enum SyntaxShape { - Any, - List, - Literal, - String, - Member, - Variable, - Number, - Path, - Pattern, - Binary, - Block, - Boolean, -} - -impl std::fmt::Display for SyntaxShape { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - SyntaxShape::Any => write!(f, "Any"), - SyntaxShape::List => write!(f, "List"), - SyntaxShape::Literal => write!(f, "Literal"), - SyntaxShape::String => write!(f, "String"), - SyntaxShape::Member => write!(f, "Member"), - SyntaxShape::Variable => write!(f, "Variable"), - SyntaxShape::Number => write!(f, "Number"), - SyntaxShape::Path => write!(f, "Path"), - SyntaxShape::Pattern => write!(f, "Pattern"), - SyntaxShape::Binary => write!(f, "Binary"), - SyntaxShape::Block => write!(f, "Block"), - SyntaxShape::Boolean => write!(f, "Boolean"), - } - } -} - -pub fn baseline_parse_next_expr( - tokens: &mut TokensIterator, - context: &Context, - source: &Text, - syntax_type: SyntaxShape, -) -> Result { - let next = tokens - .next() - .ok_or_else(|| ShellError::string("Expected token, found none"))?; - - trace!(target: "nu::parser::parse_one_expr", "syntax_type={:?}, token={:?}", syntax_type, next); - - match (syntax_type, next) { - (SyntaxShape::Path, TokenNode::Token(token)) => { - return baseline_parse_token_as_path(token, context, source) - } - - (SyntaxShape::Path, token) => { - return Err(ShellError::type_error( - "Path", - token.type_name().tagged(token.tag()), - )) - } - - (SyntaxShape::Pattern, TokenNode::Token(token)) => { - return baseline_parse_token_as_pattern(token, context, source) - } - - (SyntaxShape::Pattern, token) => { - return Err(ShellError::type_error( - "Path", - token.type_name().tagged(token.tag()), - )) - } - - (SyntaxShape::String, TokenNode::Token(token)) => { - return baseline_parse_token_as_string(token, source); - } - - (SyntaxShape::String, token) => { - return Err(ShellError::type_error( - "String", - token.type_name().tagged(token.tag()), - )) - } - - (SyntaxShape::Number, TokenNode::Token(token)) => { - return Ok(baseline_parse_token_as_number(token, source)?); - } - - (SyntaxShape::Number, token) => { - return Err(ShellError::type_error( - "Numeric", - token.type_name().tagged(token.tag()), - )) - } - - // TODO: More legit member processing - (SyntaxShape::Member, TokenNode::Token(token)) => { - return baseline_parse_token_as_string(token, source); - } - - (SyntaxShape::Member, token) => { - return Err(ShellError::type_error( - "member", - token.type_name().tagged(token.tag()), - )) - } - - (SyntaxShape::Any, _) => {} - (SyntaxShape::List, _) => {} - (SyntaxShape::Literal, _) => {} - (SyntaxShape::Variable, _) => {} - (SyntaxShape::Binary, _) => {} - (SyntaxShape::Block, _) => {} - (SyntaxShape::Boolean, _) => {} - }; - - let first = baseline_parse_semantic_token(next, context, source)?; - - let possible_op = tokens.peek(); - - let op = match possible_op { - Some(TokenNode::Operator(op)) => op.clone(), - _ => return Ok(first), - }; - - tokens.next(); - - let second = match tokens.next() { - None => { - return Err(ShellError::labeled_error( - "Expected something after an operator", - "operator", - op.tag(), - )) - } - Some(token) => baseline_parse_semantic_token(token, context, source)?, - }; - - // We definitely have a binary expression here -- let's see if we should coerce it into a block - - match syntax_type { - SyntaxShape::Any => { - let tag = first.tag().until(second.tag()); - let binary = hir::Binary::new(first, op, second); - let binary = hir::RawExpression::Binary(Box::new(binary)); - let binary = binary.tagged(tag); - - Ok(binary) - } - - SyntaxShape::Block => { - let tag = first.tag().until(second.tag()); - - let path: Tagged = match first { - Tagged { - item: hir::RawExpression::Literal(hir::Literal::Bare), - tag, - } => { - let string = tag.slice(source).to_string().tagged(tag); - let path = hir::Path::new( - // TODO: Deal with synthetic nodes that have no representation at all in source - hir::RawExpression::Variable(hir::Variable::It(Tag::unknown())) - .tagged(Tag::unknown()), - vec![string], - ); - let path = hir::RawExpression::Path(Box::new(path)); - path.tagged(first.tag()) - } - Tagged { - item: hir::RawExpression::Literal(hir::Literal::String(inner)), - tag, - } => { - let string = inner.slice(source).to_string().tagged(tag); - let path = hir::Path::new( - // TODO: Deal with synthetic nodes that have no representation at all in source - hir::RawExpression::Variable(hir::Variable::It(Tag::unknown())) - .tagged_unknown(), - vec![string], - ); - let path = hir::RawExpression::Path(Box::new(path)); - path.tagged(first.tag()) - } - Tagged { - item: hir::RawExpression::Variable(..), - .. - } => first, - Tagged { tag, item } => { - return Err(ShellError::labeled_error( - "The first part of an un-braced block must be a column name", - item.type_name(), - tag, - )) - } - }; - - let binary = hir::Binary::new(path, op, second); - let binary = hir::RawExpression::Binary(Box::new(binary)); - let binary = binary.tagged(tag); - - let block = hir::RawExpression::Block(vec![binary]); - let block = block.tagged(tag); - - Ok(block) - } - - other => Err(ShellError::unimplemented(format!( - "coerce hint {:?}", - other - ))), - } -} - -pub fn baseline_parse_semantic_token( - token: &TokenNode, - context: &Context, - source: &Text, -) -> Result { - match token { - TokenNode::Token(token) => baseline_parse_single_token(token, source), - TokenNode::Call(_call) => unimplemented!(), - TokenNode::Delimited(delimited) => baseline_parse_delimited(delimited, context, source), - TokenNode::Pipeline(_pipeline) => unimplemented!(), - TokenNode::Operator(op) => Err(ShellError::syntax_error( - "Unexpected operator".tagged(op.tag), - )), - TokenNode::Flag(flag) => Err(ShellError::syntax_error("Unexpected flag".tagged(flag.tag))), - TokenNode::Member(tag) => Err(ShellError::syntax_error( - "BUG: Top-level member".tagged(*tag), - )), - TokenNode::Whitespace(tag) => Err(ShellError::syntax_error( - "BUG: Whitespace found during parse".tagged(*tag), - )), - TokenNode::Error(error) => Err(*error.item.clone()), - TokenNode::Path(path) => baseline_parse_path(path, context, source), - } -} - -pub fn baseline_parse_delimited( - token: &Tagged, - context: &Context, - source: &Text, -) -> Result { - match token.delimiter() { - Delimiter::Brace => { - let children = token.children(); - let exprs = baseline_parse_tokens( - &mut TokensIterator::new(children), - context, - source, - SyntaxShape::Any, - )?; - - let expr = hir::RawExpression::Block(exprs); - Ok(expr.tagged(token.tag())) - } - Delimiter::Paren => unimplemented!(), - Delimiter::Square => { - let children = token.children(); - let exprs = baseline_parse_tokens( - &mut TokensIterator::new(children), - context, - source, - SyntaxShape::Any, - )?; - - let expr = hir::RawExpression::List(exprs); - Ok(expr.tagged(token.tag())) - } - } -} - -pub fn baseline_parse_path( - token: &Tagged, - context: &Context, - source: &Text, -) -> Result { - let head = baseline_parse_semantic_token(token.head(), context, source)?; - - let mut tail = vec![]; - - for part in token.tail() { - let string = match part { - TokenNode::Token(token) => match token.item() { - RawToken::Bare => token.tag().slice(source), - RawToken::String(tag) => tag.slice(source), - RawToken::Number(_) - | RawToken::Size(..) - | RawToken::Variable(_) - | RawToken::ExternalCommand(_) - | RawToken::GlobPattern - | RawToken::ExternalWord => { - return Err(ShellError::type_error( - "String", - token.type_name().tagged(part.tag()), - )) - } - }, - - TokenNode::Member(tag) => tag.slice(source), - - // TODO: Make this impossible - other => { - return Err(ShellError::syntax_error( - format!("{} in path", other.type_name()).tagged(other.tag()), - )) - } - } - .to_string(); - - tail.push(string.tagged(part.tag())); - } - - Ok(hir::path(head, tail).tagged(token.tag()).into()) -} - -#[derive(Debug, new)] -pub struct TokensIterator<'a> { - tokens: &'a [TokenNode], - #[new(default)] - index: usize, - #[new(default)] - seen: indexmap::IndexSet, -} - -impl TokensIterator<'_> { - pub fn remove(&mut self, position: usize) { - self.seen.insert(position); - } - - pub fn len(&self) -> usize { - self.tokens.len() - } - - pub fn at_end(&self) -> bool { - for index in self.index..self.tokens.len() { - if !self.seen.contains(&index) { - return false; - } - } - - true - } - - pub fn advance(&mut self) { - self.seen.insert(self.index); - self.index += 1; - } - - pub fn extract(&mut self, f: impl Fn(&TokenNode) -> Option) -> Option<(usize, T)> { - for (i, item) in self.tokens.iter().enumerate() { - if self.seen.contains(&i) { - continue; - } - - match f(item) { - None => { - continue; - } - Some(value) => { - self.seen.insert(i); - return Some((i, value)); - } - } - } - - None - } - - pub fn move_to(&mut self, pos: usize) { - self.index = pos; - } - - pub fn restart(&mut self) { - self.index = 0; - } - - pub fn clone(&self) -> TokensIterator { - TokensIterator { - tokens: self.tokens, - index: self.index, - seen: self.seen.clone(), - } - } - - pub fn peek(&self) -> Option<&TokenNode> { - let mut tokens = self.clone(); - - tokens.next() - } - - pub fn debug_remaining(&self) -> Vec { - let mut tokens = self.clone(); - tokens.restart(); - tokens.cloned().collect() - } -} - -impl<'a> Iterator for TokensIterator<'a> { - type Item = &'a TokenNode; - - fn next(&mut self) -> Option<&'a TokenNode> { - loop { - if self.index >= self.tokens.len() { - return None; - } - - if self.seen.contains(&self.index) { - self.advance(); - continue; - } - - if self.index >= self.tokens.len() { - return None; - } - - match &self.tokens[self.index] { - TokenNode::Whitespace(_) => { - self.advance(); - } - other => { - self.advance(); - return Some(other); - } - } - } - } -} diff --git a/src/parser/hir/binary.rs b/src/parser/hir/binary.rs index 02a4d416e4..a44c41d63a 100644 --- a/src/parser/hir/binary.rs +++ b/src/parser/hir/binary.rs @@ -16,6 +16,12 @@ pub struct Binary { right: Expression, } +impl fmt::Display for Binary { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "({} {} {})", self.op.as_str(), self.left, self.right) + } +} + impl ToDebug for Binary { fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { write!(f, "{}", self.left.debug(source))?; diff --git a/src/parser/hir/expand_external_tokens.rs b/src/parser/hir/expand_external_tokens.rs new file mode 100644 index 0000000000..30a2a90aaf --- /dev/null +++ b/src/parser/hir/expand_external_tokens.rs @@ -0,0 +1,87 @@ +use crate::errors::ShellError; +use crate::parser::{TokenNode, TokensIterator}; +use crate::{Tag, Tagged, Text}; + +pub fn expand_external_tokens( + token_nodes: &mut TokensIterator<'_>, + source: &Text, +) -> Result>, ShellError> { + let mut out: Vec> = vec![]; + + loop { + if let Some(tag) = expand_next_expression(token_nodes)? { + out.push(tag.tagged_string(source)); + } else { + break; + } + } + + Ok(out) +} + +pub fn expand_next_expression( + token_nodes: &mut TokensIterator<'_>, +) -> Result, ShellError> { + let first = token_nodes.next_non_ws(); + + let first = match first { + None => return Ok(None), + Some(v) => v, + }; + + let first = triage_external_head(first)?; + let mut last = first; + + loop { + let continuation = triage_continuation(token_nodes)?; + + if let Some(continuation) = continuation { + last = continuation; + } else { + break; + } + } + + Ok(Some(first.until(last))) +} + +fn triage_external_head(node: &TokenNode) -> Result { + Ok(match node { + TokenNode::Token(token) => token.tag(), + TokenNode::Call(_call) => unimplemented!(), + TokenNode::Nodes(_nodes) => unimplemented!(), + TokenNode::Delimited(_delimited) => unimplemented!(), + TokenNode::Pipeline(_pipeline) => unimplemented!(), + TokenNode::Flag(flag) => flag.tag(), + TokenNode::Member(member) => *member, + TokenNode::Whitespace(_whitespace) => { + unreachable!("This function should be called after next_non_ws()") + } + TokenNode::Error(_error) => unimplemented!(), + }) +} + +fn triage_continuation<'a, 'b>( + nodes: &'a mut TokensIterator<'b>, +) -> Result, ShellError> { + let mut peeked = nodes.peek_any(); + + let node = match peeked.node { + None => return Ok(None), + Some(node) => node, + }; + + match &node { + node if node.is_whitespace() => return Ok(None), + TokenNode::Token(..) | TokenNode::Flag(..) | TokenNode::Member(..) => {} + TokenNode::Call(..) => unimplemented!("call"), + TokenNode::Nodes(..) => unimplemented!("nodes"), + TokenNode::Delimited(..) => unimplemented!("delimited"), + TokenNode::Pipeline(..) => unimplemented!("pipeline"), + TokenNode::Whitespace(..) => unimplemented!("whitespace"), + TokenNode::Error(..) => unimplemented!("error"), + } + + peeked.commit(); + Ok(Some(node.tag())) +} diff --git a/src/parser/hir/external_command.rs b/src/parser/hir/external_command.rs index 28865330d5..2dd42c1312 100644 --- a/src/parser/hir/external_command.rs +++ b/src/parser/hir/external_command.rs @@ -9,7 +9,7 @@ use std::fmt; )] #[get = "pub(crate)"] pub struct ExternalCommand { - name: Tag, + pub(crate) name: Tag, } impl ToDebug for ExternalCommand { diff --git a/src/parser/hir/path.rs b/src/parser/hir/path.rs index f43edf1762..a1925102fb 100644 --- a/src/parser/hir/path.rs +++ b/src/parser/hir/path.rs @@ -2,19 +2,49 @@ use crate::parser::hir::Expression; use crate::prelude::*; use crate::Tagged; use derive_new::new; -use getset::Getters; +use getset::{Getters, MutGetters}; use serde::{Deserialize, Serialize}; use std::fmt; #[derive( - Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, Serialize, Deserialize, new, + Debug, + Clone, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + Getters, + MutGetters, + Serialize, + Deserialize, + new, )] #[get = "pub(crate)"] pub struct Path { head: Expression, + #[get_mut = "pub(crate)"] tail: Vec>, } +impl fmt::Display for Path { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.head)?; + + for entry in &self.tail { + write!(f, ".{}", entry.item)?; + } + + Ok(()) + } +} + +impl Path { + pub(crate) fn parts(self) -> (Expression, Vec>) { + (self.head, self.tail) + } +} + impl ToDebug for Path { fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { write!(f, "{}", self.head.debug(source))?; diff --git a/src/parser/hir/syntax_shape.rs b/src/parser/hir/syntax_shape.rs new file mode 100644 index 0000000000..5dcbd0fb76 --- /dev/null +++ b/src/parser/hir/syntax_shape.rs @@ -0,0 +1,662 @@ +mod block; +mod expression; + +use crate::cli::external_command; +use crate::commands::{classified::InternalCommand, ClassifiedCommand, Command}; +use crate::parser::hir::syntax_shape::block::AnyBlockShape; +use crate::parser::hir::tokens_iterator::Peeked; +use crate::parser::parse_command::parse_command_tail; +use crate::parser::{ + hir, + hir::{debug_tokens, TokensIterator}, + Operator, RawToken, TokenNode, +}; +use crate::prelude::*; +use derive_new::new; +use getset::Getters; +use log::trace; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; + +pub(crate) use self::expression::file_path::FilePathShape; +pub(crate) use self::expression::list::ExpressionListShape; +pub(crate) use self::expression::number::{IntShape, NumberShape}; +pub(crate) use self::expression::pattern::PatternShape; +pub(crate) use self::expression::string::StringShape; +pub(crate) use self::expression::unit::UnitShape; +pub(crate) use self::expression::variable_path::{ + ColumnPathShape, DotShape, ExpressionContinuation, ExpressionContinuationShape, MemberShape, + PathTailShape, VariablePathShape, +}; +pub(crate) use self::expression::{continue_expression, AnyExpressionShape}; + +#[derive(Debug, Copy, Clone, Serialize, Deserialize)] +pub enum SyntaxShape { + Any, + List, + String, + Member, + ColumnPath, + Number, + Int, + Path, + Pattern, + Binary, + Block, + Boolean, +} + +impl ExpandExpression for SyntaxShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + match self { + SyntaxShape::Any => expand_expr(&AnyExpressionShape, token_nodes, context), + SyntaxShape::List => Err(ShellError::unimplemented("SyntaxShape:List")), + SyntaxShape::Int => expand_expr(&IntShape, token_nodes, context), + SyntaxShape::String => expand_expr(&StringShape, token_nodes, context), + SyntaxShape::Member => { + let syntax = expand_syntax(&MemberShape, token_nodes, context)?; + Ok(syntax.to_expr()) + } + SyntaxShape::ColumnPath => { + let Tagged { item: members, tag } = + expand_syntax(&ColumnPathShape, token_nodes, context)?; + + Ok(hir::Expression::list( + members.into_iter().map(|s| s.to_expr()).collect(), + tag, + )) + } + SyntaxShape::Number => expand_expr(&NumberShape, token_nodes, context), + SyntaxShape::Path => expand_expr(&FilePathShape, token_nodes, context), + SyntaxShape::Pattern => expand_expr(&PatternShape, token_nodes, context), + SyntaxShape::Binary => Err(ShellError::unimplemented("SyntaxShape:Binary")), + SyntaxShape::Block => expand_expr(&AnyBlockShape, token_nodes, context), + SyntaxShape::Boolean => Err(ShellError::unimplemented("SyntaxShape:Boolean")), + } + } +} + +impl std::fmt::Display for SyntaxShape { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SyntaxShape::Any => write!(f, "Any"), + SyntaxShape::List => write!(f, "List"), + SyntaxShape::String => write!(f, "String"), + SyntaxShape::Int => write!(f, "Integer"), + SyntaxShape::Member => write!(f, "Member"), + SyntaxShape::ColumnPath => write!(f, "ColumnPath"), + SyntaxShape::Number => write!(f, "Number"), + SyntaxShape::Path => write!(f, "Path"), + SyntaxShape::Pattern => write!(f, "Pattern"), + SyntaxShape::Binary => write!(f, "Binary"), + SyntaxShape::Block => write!(f, "Block"), + SyntaxShape::Boolean => write!(f, "Boolean"), + } + } +} + +#[derive(Getters, new)] +pub struct ExpandContext<'context> { + #[get = "pub(crate)"] + registry: &'context CommandRegistry, + #[get = "pub(crate)"] + tag: Tag, + #[get = "pub(crate)"] + source: &'context Text, + homedir: Option, +} + +impl<'context> ExpandContext<'context> { + pub(crate) fn homedir(&self) -> Option<&Path> { + self.homedir.as_ref().map(|h| h.as_path()) + } + + #[cfg(test)] + pub fn with_empty(source: &Text, callback: impl FnOnce(ExpandContext)) { + let mut registry = CommandRegistry::new(); + registry.insert( + "ls", + crate::commands::whole_stream_command(crate::commands::LS), + ); + + callback(ExpandContext { + registry: ®istry, + tag: Tag::unknown(), + source, + homedir: None, + }) + } +} + +pub trait TestSyntax: std::fmt::Debug + Copy { + fn test<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Option>; +} + +pub trait ExpandExpression: std::fmt::Debug + Copy { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result; +} + +pub(crate) trait ExpandSyntax: std::fmt::Debug + Copy { + type Output: std::fmt::Debug; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result; +} + +pub(crate) fn expand_syntax<'a, 'b, T: ExpandSyntax>( + shape: &T, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, +) -> Result { + trace!(target: "nu::expand_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); + + let result = shape.expand_syntax(token_nodes, context); + + match result { + Err(err) => { + trace!(target: "nu::expand_syntax", "error :: {} :: {:?}", err, debug_tokens(token_nodes, context.source)); + Err(err) + } + + Ok(result) => { + trace!(target: "nu::expand_syntax", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes, context.source)); + Ok(result) + } + } +} + +pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>( + shape: &T, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, +) -> Result { + trace!(target: "nu::expand_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); + + let result = shape.expand_syntax(token_nodes, context); + + match result { + Err(err) => { + trace!(target: "nu::expand_syntax", "error :: {} :: {:?}", err, debug_tokens(token_nodes, context.source)); + Err(err) + } + + Ok(result) => { + trace!(target: "nu::expand_syntax", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes, context.source)); + Ok(result) + } + } +} + +impl ExpandSyntax for T { + type Output = hir::Expression; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + ExpandExpression::expand_expr(self, token_nodes, context) + } +} + +pub trait SkipSyntax: std::fmt::Debug + Copy { + fn skip<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result<(), ShellError>; +} + +enum BarePathState { + Initial, + Seen(Tag, Tag), + Error(ShellError), +} + +impl BarePathState { + pub fn seen(self, tag: Tag) -> BarePathState { + match self { + BarePathState::Initial => BarePathState::Seen(tag, tag), + BarePathState::Seen(start, _) => BarePathState::Seen(start, tag), + BarePathState::Error(err) => BarePathState::Error(err), + } + } + + pub fn end(self, peeked: Peeked, reason: impl Into) -> BarePathState { + match self { + BarePathState::Initial => BarePathState::Error(peeked.type_error(reason)), + BarePathState::Seen(start, end) => BarePathState::Seen(start, end), + BarePathState::Error(err) => BarePathState::Error(err), + } + } + + pub fn into_bare(self) -> Result { + match self { + BarePathState::Initial => unreachable!("into_bare in initial state"), + BarePathState::Seen(start, end) => Ok(start.until(end)), + BarePathState::Error(err) => Err(err), + } + } +} + +pub fn expand_bare<'a, 'b>( + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + predicate: impl Fn(&TokenNode) -> bool, +) -> Result { + let mut state = BarePathState::Initial; + + loop { + // Whitespace ends a word + let mut peeked = token_nodes.peek_any(); + + match peeked.node { + None => { + state = state.end(peeked, "word"); + break; + } + Some(node) => { + if predicate(node) { + state = state.seen(node.tag()); + peeked.commit(); + } else { + state = state.end(peeked, "word"); + break; + } + } + } + } + + state.into_bare() +} + +#[derive(Debug, Copy, Clone)] +pub struct BarePathShape; + +impl ExpandSyntax for BarePathShape { + type Output = Tag; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + expand_bare(token_nodes, context, |token| match token { + TokenNode::Token(Tagged { + item: RawToken::Bare, + .. + }) + | TokenNode::Token(Tagged { + item: RawToken::Operator(Operator::Dot), + .. + }) => true, + + _ => false, + }) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct BareShape; + +impl ExpandSyntax for BareShape { + type Output = Tagged; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let peeked = token_nodes.peek_any().not_eof("word")?; + + match peeked.node { + TokenNode::Token(Tagged { + item: RawToken::Bare, + tag, + }) => { + peeked.commit(); + Ok(tag.tagged_string(context.source)) + } + + other => Err(ShellError::type_error("word", other.tagged_type_name())), + } + } +} + +impl TestSyntax for BareShape { + fn test<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + ) -> Option> { + let peeked = token_nodes.peek_any(); + + match peeked.node { + Some(TokenNode::Token(token)) => match token.item { + RawToken::Bare => Some(peeked), + _ => None, + }, + + _ => None, + } + } +} + +#[derive(Debug)] +pub enum CommandSignature { + Internal(Tagged>), + LiteralExternal { outer: Tag, inner: Tag }, + External(Tag), + Expression(hir::Expression), +} + +impl CommandSignature { + pub fn to_expression(&self) -> hir::Expression { + match self { + CommandSignature::Internal(command) => { + let tag = command.tag; + hir::RawExpression::Command(tag).tagged(tag) + } + CommandSignature::LiteralExternal { outer, inner } => { + hir::RawExpression::ExternalCommand(hir::ExternalCommand::new(*inner)).tagged(outer) + } + CommandSignature::External(tag) => { + hir::RawExpression::ExternalCommand(hir::ExternalCommand::new(*tag)).tagged(tag) + } + CommandSignature::Expression(expr) => expr.clone(), + } + } +} + +#[derive(Debug, Copy, Clone)] +pub struct CommandHeadShape; + +impl ExpandSyntax for CommandHeadShape { + type Output = CommandSignature; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + let node = + parse_single_node_skipping_ws(token_nodes, "command head1", |token, token_tag| { + Ok(match token { + RawToken::ExternalCommand(tag) => CommandSignature::LiteralExternal { + outer: token_tag, + inner: tag, + }, + RawToken::Bare => { + let name = token_tag.slice(context.source); + if context.registry.has(name) { + let command = context.registry.expect_command(name); + CommandSignature::Internal(command.tagged(token_tag)) + } else { + CommandSignature::External(token_tag) + } + } + _ => { + return Err(ShellError::type_error( + "command head2", + token.type_name().tagged(token_tag), + )) + } + }) + }); + + match node { + Ok(expr) => return Ok(expr), + Err(_) => match expand_expr(&AnyExpressionShape, token_nodes, context) { + Ok(expr) => return Ok(CommandSignature::Expression(expr)), + Err(_) => Err(token_nodes.peek_non_ws().type_error("command head3")), + }, + } + } +} + +#[derive(Debug, Copy, Clone)] +pub struct ClassifiedCommandShape; + +impl ExpandSyntax for ClassifiedCommandShape { + type Output = ClassifiedCommand; + + fn expand_syntax<'a, 'b>( + &self, + iterator: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let head = expand_syntax(&CommandHeadShape, iterator, context)?; + + match &head { + CommandSignature::Expression(expr) => Err(ShellError::syntax_error( + "Unexpected expression in command position".tagged(expr.tag), + )), + + // If the command starts with `^`, treat it as an external command no matter what + CommandSignature::External(name) => { + let name_str = name.slice(&context.source); + + external_command(iterator, &context.source, name_str.tagged(name)) + } + + CommandSignature::LiteralExternal { outer, inner } => { + let name_str = inner.slice(&context.source); + + external_command(iterator, &context.source, name_str.tagged(outer)) + } + + CommandSignature::Internal(command) => { + let tail = + parse_command_tail(&command.signature(), &context, iterator, command.tag)?; + + let (positional, named) = match tail { + None => (None, None), + Some((positional, named)) => (positional, named), + }; + + let call = hir::Call { + head: Box::new(head.to_expression()), + positional, + named, + }; + + Ok(ClassifiedCommand::Internal(InternalCommand::new( + command.item.name().to_string(), + command.tag, + call, + ))) + } + } + } +} + +#[derive(Debug, Copy, Clone)] +pub struct InternalCommandHeadShape; + +impl ExpandExpression for InternalCommandHeadShape { + fn expand_expr( + &self, + token_nodes: &mut TokensIterator<'_>, + _context: &ExpandContext, + ) -> Result { + let peeked_head = token_nodes.peek_non_ws().not_eof("command head4")?; + + let expr = match peeked_head.node { + TokenNode::Token( + spanned @ Tagged { + item: RawToken::Bare, + .. + }, + ) => spanned.map(|_| hir::RawExpression::Literal(hir::Literal::Bare)), + + TokenNode::Token(Tagged { + item: RawToken::String(inner_tag), + tag, + }) => hir::RawExpression::Literal(hir::Literal::String(*inner_tag)).tagged(*tag), + + node => { + return Err(ShellError::type_error( + "command head5", + node.tagged_type_name(), + )) + } + }; + + peeked_head.commit(); + + Ok(expr) + } +} + +fn parse_single_node<'a, 'b, T>( + token_nodes: &'b mut TokensIterator<'a>, + expected: &'static str, + callback: impl FnOnce(RawToken, Tag) -> Result, +) -> Result { + let peeked = token_nodes.peek_any().not_eof(expected)?; + + let expr = match peeked.node { + TokenNode::Token(token) => callback(token.item, token.tag())?, + + other => return Err(ShellError::type_error(expected, other.tagged_type_name())), + }; + + peeked.commit(); + + Ok(expr) +} + +fn parse_single_node_skipping_ws<'a, 'b, T>( + token_nodes: &'b mut TokensIterator<'a>, + expected: &'static str, + callback: impl FnOnce(RawToken, Tag) -> Result, +) -> Result { + let peeked = token_nodes.peek_non_ws().not_eof(expected)?; + + let expr = match peeked.node { + TokenNode::Token(token) => callback(token.item, token.tag())?, + + other => return Err(ShellError::type_error(expected, other.tagged_type_name())), + }; + + peeked.commit(); + + Ok(expr) +} + +#[derive(Debug, Copy, Clone)] +pub struct WhitespaceShape; + +impl ExpandSyntax for WhitespaceShape { + type Output = Tag; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + ) -> Result { + let peeked = token_nodes.peek_any().not_eof("whitespace")?; + + let tag = match peeked.node { + TokenNode::Whitespace(tag) => *tag, + + other => { + return Err(ShellError::type_error( + "whitespace", + other.tagged_type_name(), + )) + } + }; + + peeked.commit(); + + Ok(tag) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct SpacedExpression { + inner: T, +} + +impl ExpandExpression for SpacedExpression { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + // TODO: Make the name part of the trait + let peeked = token_nodes.peek_any().not_eof("whitespace")?; + + match peeked.node { + TokenNode::Whitespace(_) => { + peeked.commit(); + expand_expr(&self.inner, token_nodes, context) + } + + other => Err(ShellError::type_error( + "whitespace", + other.tagged_type_name(), + )), + } + } +} + +pub fn maybe_spaced(inner: T) -> MaybeSpacedExpression { + MaybeSpacedExpression { inner } +} + +#[derive(Debug, Copy, Clone)] +pub struct MaybeSpacedExpression { + inner: T, +} + +impl ExpandExpression for MaybeSpacedExpression { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + // TODO: Make the name part of the trait + let peeked = token_nodes.peek_any().not_eof("whitespace")?; + + match peeked.node { + TokenNode::Whitespace(_) => { + peeked.commit(); + expand_expr(&self.inner, token_nodes, context) + } + + _ => { + peeked.rollback(); + expand_expr(&self.inner, token_nodes, context) + } + } + } +} + +pub fn spaced(inner: T) -> SpacedExpression { + SpacedExpression { inner } +} + +fn expand_variable(tag: Tag, token_tag: Tag, source: &Text) -> hir::Expression { + if tag.slice(source) == "it" { + hir::Expression::it_variable(tag, token_tag) + } else { + hir::Expression::variable(tag, token_tag) + } +} diff --git a/src/parser/hir/syntax_shape/block.rs b/src/parser/hir/syntax_shape/block.rs new file mode 100644 index 0000000000..a78292b34e --- /dev/null +++ b/src/parser/hir/syntax_shape/block.rs @@ -0,0 +1,168 @@ +use crate::errors::ShellError; +use crate::parser::{ + hir, + hir::syntax_shape::{ + continue_expression, expand_expr, expand_syntax, ExpandContext, ExpandExpression, + ExpressionListShape, PathTailShape, VariablePathShape, + }, + hir::tokens_iterator::TokensIterator, + RawToken, TokenNode, +}; +use crate::{Tag, Tagged, TaggedItem}; + +#[derive(Debug, Copy, Clone)] +pub struct AnyBlockShape; + +impl ExpandExpression for AnyBlockShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + let block = token_nodes.peek_non_ws().not_eof("block")?; + + // is it just a block? + let block = block.node.as_block(); + + match block { + Some(block) => { + let mut iterator = TokensIterator::new(&block.item, context.tag, false); + + let exprs = expand_syntax(&ExpressionListShape, &mut iterator, context)?; + + return Ok(hir::RawExpression::Block(exprs).tagged(block.tag)); + } + _ => {} + } + + expand_syntax(&ShorthandBlock, token_nodes, context) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct ShorthandBlock; + +impl ExpandExpression for ShorthandBlock { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let path = expand_expr(&ShorthandPath, token_nodes, context)?; + let start = path.tag; + let expr = continue_expression(path, token_nodes, context)?; + let end = expr.tag; + let block = hir::RawExpression::Block(vec![expr]).tagged(start.until(end)); + + Ok(block) + } +} + +/// A shorthand for `$it.foo."bar"`, used inside of a shorthand block +#[derive(Debug, Copy, Clone)] +pub struct ShorthandPath; + +impl ExpandExpression for ShorthandPath { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + // if it's a variable path, that's the head part + let path = expand_expr(&VariablePathShape, token_nodes, context); + + match path { + Ok(path) => return Ok(path), + Err(_) => {} + } + + // Synthesize the head of the shorthand path (`` -> `$it.`) + let mut head = expand_expr(&ShorthandHeadShape, token_nodes, context)?; + + // Now that we've synthesized the head, of the path, proceed to expand the tail of the path + // like any other path. + let tail = expand_syntax(&PathTailShape, token_nodes, context); + + match tail { + Err(_) => return Ok(head), + Ok((tail, _)) => { + // For each member that `PathTailShape` expanded, join it onto the existing expression + // to form a new path + for member in tail { + head = hir::Expression::dot_member(head, member); + } + + println!("{:?}", head); + + Ok(head) + } + } + } +} + +/// A shorthand for `$it.foo."bar"`, used inside of a shorthand block +#[derive(Debug, Copy, Clone)] +pub struct ShorthandHeadShape; + +impl ExpandExpression for ShorthandHeadShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + // A shorthand path must not be at EOF + let peeked = token_nodes.peek_non_ws().not_eof("shorthand path")?; + + match peeked.node { + // If the head of a shorthand path is a bare token, it expands to `$it.bare` + TokenNode::Token(Tagged { + item: RawToken::Bare, + tag, + }) => { + // Commit the peeked token + peeked.commit(); + + // Synthesize an `$it` expression + let it = synthetic_it(token_nodes.anchor()); + + // Make a path out of `$it` and the bare token as a member + Ok(hir::Expression::path( + it, + vec![tag.tagged_string(context.source)], + tag, + )) + } + + // If the head of a shorthand path is a string, it expands to `$it."some string"` + TokenNode::Token(Tagged { + item: RawToken::String(inner), + tag: outer, + }) => { + // Commit the peeked token + peeked.commit(); + + // Synthesize an `$it` expression + let it = synthetic_it(token_nodes.anchor()); + + // Make a path out of `$it` and the bare token as a member + Ok(hir::Expression::path( + it, + vec![inner.string(context.source).tagged(outer)], + outer, + )) + } + + // Any other token is not a valid bare head + other => { + return Err(ShellError::type_error( + "shorthand path", + other.tagged_type_name(), + )) + } + } + } +} + +fn synthetic_it(origin: uuid::Uuid) -> hir::Expression { + hir::Expression::it_variable(Tag::unknown_span(origin), Tag::unknown_span(origin)) +} diff --git a/src/parser/hir/syntax_shape/expression.rs b/src/parser/hir/syntax_shape/expression.rs new file mode 100644 index 0000000000..58cfa4a1a5 --- /dev/null +++ b/src/parser/hir/syntax_shape/expression.rs @@ -0,0 +1,188 @@ +pub(crate) mod delimited; +pub(crate) mod file_path; +pub(crate) mod list; +pub(crate) mod number; +pub(crate) mod pattern; +pub(crate) mod string; +pub(crate) mod unit; +pub(crate) mod variable_path; + +use crate::parser::hir::syntax_shape::{ + expand_expr, expand_syntax, expand_variable, expression::delimited::expand_delimited_expr, + BareShape, DotShape, ExpandContext, ExpandExpression, ExpandSyntax, ExpressionContinuation, + ExpressionContinuationShape, UnitShape, +}; +use crate::parser::{ + hir, + hir::{Expression, Operator, TokensIterator}, + RawToken, Token, TokenNode, +}; +use crate::prelude::*; +use std::path::PathBuf; + +#[derive(Debug, Copy, Clone)] +pub struct AnyExpressionShape; + +impl ExpandExpression for AnyExpressionShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + // Look for an expression at the cursor + let head = expand_expr(&AnyExpressionStartShape, token_nodes, context)?; + + continue_expression(head, token_nodes, context) + } +} + +pub(crate) fn continue_expression( + mut head: hir::Expression, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, +) -> Result { + loop { + // Check to see whether there's any continuation after the head expression + let continuation = expand_syntax(&ExpressionContinuationShape, token_nodes, context); + + match continuation { + // If there's no continuation, return the head + Err(_) => return Ok(head), + // Otherwise, form a new expression by combining the head with the continuation + Ok(continuation) => match continuation { + // If the continuation is a `.member`, form a path with the new member + ExpressionContinuation::DotSuffix(_dot, member) => { + head = Expression::dot_member(head, member); + } + + // Otherwise, if the continuation is an infix suffix, form an infix expression + ExpressionContinuation::InfixSuffix(op, expr) => { + head = Expression::infix(head, op, expr); + } + }, + } + } +} + +#[derive(Debug, Copy, Clone)] +pub struct AnyExpressionStartShape; + +impl ExpandExpression for AnyExpressionStartShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + let size = expand_expr(&UnitShape, token_nodes, context); + + match size { + Ok(expr) => return Ok(expr), + Err(_) => {} + } + + let peek_next = token_nodes.peek_any().not_eof("expression")?; + + let head = match peek_next.node { + TokenNode::Token(token) => match token.item { + RawToken::Bare | RawToken::Operator(Operator::Dot) => { + let start = token.tag; + peek_next.commit(); + + let end = expand_syntax(&BareTailShape, token_nodes, context)?; + + match end { + Some(end) => return Ok(hir::Expression::bare(start.until(end))), + None => return Ok(hir::Expression::bare(start)), + } + } + _ => { + peek_next.commit(); + expand_one_context_free_token(*token, context) + } + }, + node @ TokenNode::Call(_) + | node @ TokenNode::Nodes(_) + | node @ TokenNode::Pipeline(_) + | node @ TokenNode::Flag(_) + | node @ TokenNode::Member(_) + | node @ TokenNode::Whitespace(_) => { + return Err(ShellError::type_error( + "expression", + node.tagged_type_name(), + )) + } + TokenNode::Delimited(delimited) => { + peek_next.commit(); + expand_delimited_expr(delimited, context) + } + + TokenNode::Error(error) => return Err(*error.item.clone()), + }?; + + Ok(head) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct BareTailShape; + +impl ExpandSyntax for BareTailShape { + type Output = Option; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result, ShellError> { + let mut end: Option = None; + + loop { + match expand_syntax(&BareShape, token_nodes, context) { + Ok(bare) => { + end = Some(bare.tag); + continue; + } + + Err(_) => match expand_syntax(&DotShape, token_nodes, context) { + Ok(dot) => { + end = Some(dot); + continue; + } + + Err(_) => break, + }, + } + } + + Ok(end) + } +} + +fn expand_one_context_free_token<'a, 'b>( + token: Token, + context: &ExpandContext, +) -> Result { + Ok(match token.item { + RawToken::Number(number) => { + hir::Expression::number(number.to_number(context.source), token.tag) + } + RawToken::Operator(..) => { + return Err(ShellError::syntax_error( + "unexpected operator, expected an expression".tagged(token.tag), + )) + } + RawToken::Size(..) => unimplemented!("size"), + RawToken::String(tag) => hir::Expression::string(tag, token.tag), + RawToken::Variable(tag) => expand_variable(tag, token.tag, &context.source), + RawToken::ExternalCommand(_) => unimplemented!(), + RawToken::ExternalWord => unimplemented!(), + RawToken::GlobPattern => hir::Expression::pattern(token.tag), + RawToken::Bare => hir::Expression::string(token.tag, token.tag), + }) +} + +pub fn expand_file_path(string: &str, context: &ExpandContext) -> PathBuf { + let expanded = shellexpand::tilde_with_context(string, || context.homedir()); + + PathBuf::from(expanded.as_ref()) +} diff --git a/src/parser/hir/syntax_shape/expression/delimited.rs b/src/parser/hir/syntax_shape/expression/delimited.rs new file mode 100644 index 0000000000..0a01b0fc26 --- /dev/null +++ b/src/parser/hir/syntax_shape/expression/delimited.rs @@ -0,0 +1,38 @@ +use crate::parser::hir::syntax_shape::{expand_syntax, ExpandContext, ExpressionListShape}; +use crate::parser::{hir, hir::TokensIterator}; +use crate::parser::{DelimitedNode, Delimiter}; +use crate::prelude::*; + +pub fn expand_delimited_expr( + delimited: &Tagged, + context: &ExpandContext, +) -> Result { + match &delimited.item { + DelimitedNode { + delimiter: Delimiter::Square, + children, + } => { + let mut tokens = TokensIterator::new(&children, delimited.tag, false); + + let list = expand_syntax(&ExpressionListShape, &mut tokens, context); + + Ok(hir::Expression::list(list?, delimited.tag)) + } + + DelimitedNode { + delimiter: Delimiter::Paren, + .. + } => Err(ShellError::type_error( + "expression", + "unimplemented call expression".tagged(delimited.tag), + )), + + DelimitedNode { + delimiter: Delimiter::Brace, + .. + } => Err(ShellError::type_error( + "expression", + "unimplemented block expression".tagged(delimited.tag), + )), + } +} diff --git a/src/parser/hir/syntax_shape/expression/file_path.rs b/src/parser/hir/syntax_shape/expression/file_path.rs new file mode 100644 index 0000000000..c0e5c7c2ab --- /dev/null +++ b/src/parser/hir/syntax_shape/expression/file_path.rs @@ -0,0 +1,59 @@ +use crate::parser::hir::syntax_shape::{ + expand_syntax, expression::expand_file_path, parse_single_node, BarePathShape, ExpandContext, + ExpandExpression, +}; +use crate::parser::{hir, hir::TokensIterator, RawToken}; +use crate::prelude::*; + +#[derive(Debug, Copy, Clone)] +pub struct FilePathShape; + +impl ExpandExpression for FilePathShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + let bare = expand_syntax(&BarePathShape, token_nodes, context); + + match bare { + Ok(tag) => { + let string = tag.slice(context.source); + let path = expand_file_path(string, context); + return Ok(hir::Expression::file_path(path, tag)); + } + Err(_) => {} + } + + parse_single_node(token_nodes, "Path", |token, token_tag| { + Ok(match token { + RawToken::GlobPattern => { + return Err(ShellError::type_error( + "Path", + "glob pattern".tagged(token_tag), + )) + } + RawToken::Operator(..) => { + return Err(ShellError::type_error("Path", "operator".tagged(token_tag))) + } + RawToken::Variable(tag) if tag.slice(context.source) == "it" => { + hir::Expression::it_variable(tag, token_tag) + } + RawToken::Variable(tag) => hir::Expression::variable(tag, token_tag), + RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token_tag), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token_tag)), + RawToken::Number(_) => hir::Expression::bare(token_tag), + RawToken::Size(_, _) => hir::Expression::bare(token_tag), + RawToken::Bare => hir::Expression::file_path( + expand_file_path(token_tag.slice(context.source), context), + token_tag, + ), + + RawToken::String(tag) => hir::Expression::file_path( + expand_file_path(tag.slice(context.source), context), + token_tag, + ), + }) + }) + } +} diff --git a/src/parser/hir/syntax_shape/expression/list.rs b/src/parser/hir/syntax_shape/expression/list.rs new file mode 100644 index 0000000000..9d28f44141 --- /dev/null +++ b/src/parser/hir/syntax_shape/expression/list.rs @@ -0,0 +1,43 @@ +use crate::errors::ShellError; +use crate::parser::{ + hir, + hir::syntax_shape::{ + expand_expr, maybe_spaced, spaced, AnyExpressionShape, ExpandContext, ExpandSyntax, + }, + hir::{debug_tokens, TokensIterator}, +}; + +#[derive(Debug, Copy, Clone)] +pub struct ExpressionListShape; + +impl ExpandSyntax for ExpressionListShape { + type Output = Vec; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result, ShellError> { + let mut exprs = vec![]; + + if token_nodes.at_end_possible_ws() { + return Ok(exprs); + } + + let expr = expand_expr(&maybe_spaced(AnyExpressionShape), token_nodes, context)?; + + exprs.push(expr); + + println!("{:?}", debug_tokens(token_nodes, context.source)); + + loop { + if token_nodes.at_end_possible_ws() { + return Ok(exprs); + } + + let expr = expand_expr(&spaced(AnyExpressionShape), token_nodes, context)?; + + exprs.push(expr); + } + } +} diff --git a/src/parser/hir/syntax_shape/expression/number.rs b/src/parser/hir/syntax_shape/expression/number.rs new file mode 100644 index 0000000000..5b77044a2d --- /dev/null +++ b/src/parser/hir/syntax_shape/expression/number.rs @@ -0,0 +1,97 @@ +use crate::parser::hir::syntax_shape::{parse_single_node, ExpandContext, ExpandExpression}; +use crate::parser::{ + hir, + hir::{RawNumber, TokensIterator}, + RawToken, +}; +use crate::prelude::*; + +#[derive(Debug, Copy, Clone)] +pub struct NumberShape; + +impl ExpandExpression for NumberShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + parse_single_node(token_nodes, "Number", |token, token_tag| { + Ok(match token { + RawToken::GlobPattern => { + return Err(ShellError::type_error( + "Number", + "glob pattern".to_string().tagged(token_tag), + )) + } + RawToken::Operator(..) => { + return Err(ShellError::type_error( + "Number", + "operator".to_string().tagged(token_tag), + )) + } + RawToken::Variable(tag) if tag.slice(context.source) == "it" => { + hir::Expression::it_variable(tag, token_tag) + } + RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token_tag), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token_tag)), + RawToken::Variable(tag) => hir::Expression::variable(tag, token_tag), + RawToken::Number(number) => { + hir::Expression::number(number.to_number(context.source), token_tag) + } + RawToken::Size(number, unit) => { + hir::Expression::size(number.to_number(context.source), unit, token_tag) + } + RawToken::Bare => hir::Expression::bare(token_tag), + RawToken::String(tag) => hir::Expression::string(tag, token_tag), + }) + }) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct IntShape; + +impl ExpandExpression for IntShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + parse_single_node(token_nodes, "Integer", |token, token_tag| { + Ok(match token { + RawToken::GlobPattern => { + return Err(ShellError::type_error( + "Integer", + "glob pattern".to_string().tagged(token_tag), + )) + } + RawToken::Operator(..) => { + return Err(ShellError::type_error( + "Integer", + "operator".to_string().tagged(token_tag), + )) + } + RawToken::Variable(tag) if tag.slice(context.source) == "it" => { + hir::Expression::it_variable(tag, token_tag) + } + RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token_tag), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token_tag)), + RawToken::Variable(tag) => hir::Expression::variable(tag, token_tag), + RawToken::Number(number @ RawNumber::Int(_)) => { + hir::Expression::number(number.to_number(context.source), token_tag) + } + token @ RawToken::Number(_) => { + return Err(ShellError::type_error( + "Integer", + token.type_name().tagged(token_tag), + )); + } + RawToken::Size(number, unit) => { + hir::Expression::size(number.to_number(context.source), unit, token_tag) + } + RawToken::Bare => hir::Expression::bare(token_tag), + RawToken::String(tag) => hir::Expression::string(tag, token_tag), + }) + }) + } +} diff --git a/src/parser/hir/syntax_shape/expression/pattern.rs b/src/parser/hir/syntax_shape/expression/pattern.rs new file mode 100644 index 0000000000..4105b79b4f --- /dev/null +++ b/src/parser/hir/syntax_shape/expression/pattern.rs @@ -0,0 +1,86 @@ +use crate::parser::hir::syntax_shape::{ + expand_bare, expand_syntax, expression::expand_file_path, parse_single_node, ExpandContext, + ExpandExpression, ExpandSyntax, +}; +use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode}; +use crate::prelude::*; + +#[derive(Debug, Copy, Clone)] +pub struct PatternShape; + +impl ExpandExpression for PatternShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + let pattern = expand_syntax(&BarePatternShape, token_nodes, context); + + match pattern { + Ok(tag) => { + return Ok(hir::Expression::pattern(tag)); + } + Err(_) => {} + } + + parse_single_node(token_nodes, "Pattern", |token, token_tag| { + Ok(match token { + RawToken::GlobPattern => { + return Err(ShellError::unreachable( + "glob pattern after glob already returned", + )) + } + RawToken::Operator(..) => { + return Err(ShellError::unreachable("dot after glob already returned")) + } + RawToken::Bare => { + return Err(ShellError::unreachable("bare after glob already returned")) + } + + RawToken::Variable(tag) if tag.slice(context.source) == "it" => { + hir::Expression::it_variable(tag, token_tag) + } + RawToken::Variable(tag) => hir::Expression::variable(tag, token_tag), + RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token_tag), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token_tag)), + RawToken::Number(_) => hir::Expression::bare(token_tag), + RawToken::Size(_, _) => hir::Expression::bare(token_tag), + + RawToken::String(tag) => hir::Expression::file_path( + expand_file_path(tag.slice(context.source), context), + token_tag, + ), + }) + }) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct BarePatternShape; + +impl ExpandSyntax for BarePatternShape { + type Output = Tag; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + expand_bare(token_nodes, context, |token| match token { + TokenNode::Token(Tagged { + item: RawToken::Bare, + .. + }) + | TokenNode::Token(Tagged { + item: RawToken::Operator(Operator::Dot), + .. + }) + | TokenNode::Token(Tagged { + item: RawToken::GlobPattern, + .. + }) => true, + + _ => false, + }) + } +} diff --git a/src/parser/hir/syntax_shape/expression/string.rs b/src/parser/hir/syntax_shape/expression/string.rs new file mode 100644 index 0000000000..6a4973febe --- /dev/null +++ b/src/parser/hir/syntax_shape/expression/string.rs @@ -0,0 +1,60 @@ +use crate::parser::hir::syntax_shape::{ + expand_variable, parse_single_node, ExpandContext, ExpandExpression, TestSyntax, +}; +use crate::parser::hir::tokens_iterator::Peeked; +use crate::parser::{hir, hir::TokensIterator, RawToken, TokenNode}; +use crate::prelude::*; + +#[derive(Debug, Copy, Clone)] +pub struct StringShape; + +impl ExpandExpression for StringShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + parse_single_node(token_nodes, "String", |token, token_tag| { + Ok(match token { + RawToken::GlobPattern => { + return Err(ShellError::type_error( + "String", + "glob pattern".tagged(token_tag), + )) + } + RawToken::Operator(..) => { + return Err(ShellError::type_error( + "String", + "operator".tagged(token_tag), + )) + } + RawToken::Variable(tag) => expand_variable(tag, token_tag, &context.source), + RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token_tag), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token_tag)), + RawToken::Number(_) => hir::Expression::bare(token_tag), + RawToken::Size(_, _) => hir::Expression::bare(token_tag), + RawToken::Bare => hir::Expression::bare(token_tag), + RawToken::String(tag) => hir::Expression::string(tag, token_tag), + }) + }) + } +} + +impl TestSyntax for StringShape { + fn test<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + ) -> Option> { + let peeked = token_nodes.peek_any(); + + match peeked.node { + Some(TokenNode::Token(token)) => match token.item { + RawToken::String(_) => Some(peeked), + _ => None, + }, + + _ => None, + } + } +} diff --git a/src/parser/hir/syntax_shape/expression/unit.rs b/src/parser/hir/syntax_shape/expression/unit.rs new file mode 100644 index 0000000000..cc3642bda5 --- /dev/null +++ b/src/parser/hir/syntax_shape/expression/unit.rs @@ -0,0 +1,89 @@ +use crate::parser::hir::syntax_shape::{ExpandContext, ExpandExpression}; +use crate::parser::parse::tokens::RawNumber; +use crate::parser::parse::unit::Unit; +use crate::parser::{hir, hir::TokensIterator, RawToken, TokenNode}; +use crate::prelude::*; +use nom::branch::alt; +use nom::bytes::complete::tag; +use nom::character::complete::digit1; +use nom::combinator::{all_consuming, opt, value}; +use nom::IResult; + +#[derive(Debug, Copy, Clone)] +pub struct UnitShape; + +impl ExpandExpression for UnitShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let peeked = token_nodes.peek_any().not_eof("unit")?; + + let tag = match peeked.node { + TokenNode::Token(Tagged { + item: RawToken::Bare, + tag, + }) => tag, + _ => return Err(peeked.type_error("unit")), + }; + + let unit = unit_size(tag.slice(context.source), *tag); + + let (_, (number, unit)) = match unit { + Err(_) => { + return Err(ShellError::type_error( + "unit", + "word".tagged(Tag::unknown()), + )) + } + Ok((number, unit)) => (number, unit), + }; + + Ok(hir::Expression::size( + number.to_number(context.source), + unit, + tag, + )) + } +} + +fn unit_size(input: &str, bare_tag: Tag) -> IResult<&str, (Tagged, Unit)> { + let (input, digits) = digit1(input)?; + + let (input, dot) = opt(tag("."))(input)?; + + let (input, number) = match dot { + Some(dot) => { + let (input, rest) = digit1(input)?; + ( + input, + RawNumber::decimal(( + bare_tag.span.start(), + bare_tag.span.start() + digits.len() + dot.len() + rest.len(), + bare_tag.anchor, + )), + ) + } + + None => ( + input, + RawNumber::int(( + bare_tag.span.start(), + bare_tag.span.start() + digits.len(), + bare_tag.anchor, + )), + ), + }; + + let (input, unit) = all_consuming(alt(( + value(Unit::B, alt((tag("B"), tag("b")))), + value(Unit::KB, alt((tag("KB"), tag("kb"), tag("Kb")))), + value(Unit::MB, alt((tag("MB"), tag("mb"), tag("Mb")))), + value(Unit::MB, alt((tag("GB"), tag("gb"), tag("Gb")))), + value(Unit::MB, alt((tag("TB"), tag("tb"), tag("Tb")))), + value(Unit::MB, alt((tag("PB"), tag("pb"), tag("Pb")))), + )))(input)?; + + Ok((input, (number, unit))) +} diff --git a/src/parser/hir/syntax_shape/expression/variable_path.rs b/src/parser/hir/syntax_shape/expression/variable_path.rs new file mode 100644 index 0000000000..afea1b1499 --- /dev/null +++ b/src/parser/hir/syntax_shape/expression/variable_path.rs @@ -0,0 +1,396 @@ +use crate::parser::hir::syntax_shape::{ + expand_expr, expand_syntax, parse_single_node, AnyExpressionShape, BareShape, ExpandContext, + ExpandExpression, ExpandSyntax, Peeked, SkipSyntax, StringShape, TestSyntax, WhitespaceShape, +}; +use crate::parser::{hir, hir::Expression, hir::TokensIterator, Operator, RawToken}; +use crate::prelude::*; + +#[derive(Debug, Copy, Clone)] +pub struct VariablePathShape; + +impl ExpandExpression for VariablePathShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + // 1. let the head be the first token, expecting a variable + // 2. let the tail be an empty list of members + // 2. while the next token (excluding ws) is a dot: + // 1. consume the dot + // 2. consume the next token as a member and push it onto tail + + let head = expand_expr(&VariableShape, token_nodes, context)?; + let start = head.tag(); + let mut end = start; + let mut tail: Vec> = vec![]; + + loop { + match DotShape.skip(token_nodes, context) { + Err(_) => break, + Ok(_) => {} + } + + let syntax = expand_syntax(&MemberShape, token_nodes, context)?; + let member = syntax.to_tagged_string(context.source); + + end = member.tag(); + tail.push(member); + } + + Ok(hir::Expression::path(head, tail, start.until(end))) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct PathTailShape; + +impl ExpandSyntax for PathTailShape { + type Output = (Vec>, Tag); + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let mut end: Option = None; + let mut tail = vec![]; + + loop { + match DotShape.skip(token_nodes, context) { + Err(_) => break, + Ok(_) => {} + } + + let syntax = expand_syntax(&MemberShape, token_nodes, context)?; + let member = syntax.to_tagged_string(context.source); + end = Some(member.tag()); + tail.push(member); + } + + match end { + None => { + return Err(ShellError::type_error( + "path tail", + token_nodes.typed_tag_at_cursor(), + )) + } + + Some(end) => Ok((tail, end)), + } + } +} + +#[derive(Debug)] +pub enum ExpressionContinuation { + DotSuffix(Tag, Tagged), + InfixSuffix(Tagged, Expression), +} + +/// An expression continuation +#[derive(Debug, Copy, Clone)] +pub struct ExpressionContinuationShape; + +impl ExpandSyntax for ExpressionContinuationShape { + type Output = ExpressionContinuation; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + // Try to expand a `.` + let dot = expand_syntax(&DotShape, token_nodes, context); + + match dot { + // If a `.` was matched, it's a `Path`, and we expect a `Member` next + Ok(dot) => { + let syntax = expand_syntax(&MemberShape, token_nodes, context)?; + let member = syntax.to_tagged_string(context.source); + + Ok(ExpressionContinuation::DotSuffix(dot, member)) + } + + // Otherwise, we expect an infix operator and an expression next + Err(_) => { + let (_, op, _) = expand_syntax(&InfixShape, token_nodes, context)?; + let next = expand_expr(&AnyExpressionShape, token_nodes, context)?; + + Ok(ExpressionContinuation::InfixSuffix(op, next)) + } + } + } +} + +#[derive(Debug, Copy, Clone)] +pub struct VariableShape; + +impl ExpandExpression for VariableShape { + fn expand_expr<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + parse_single_node(token_nodes, "variable", |token, token_tag| { + Ok(match token { + RawToken::Variable(tag) => { + if tag.slice(context.source) == "it" { + hir::Expression::it_variable(tag, token_tag) + } else { + hir::Expression::variable(tag, token_tag) + } + } + _ => { + return Err(ShellError::type_error( + "variable", + token.type_name().tagged(token_tag), + )) + } + }) + }) + } +} + +#[derive(Debug, Clone, Copy)] +pub enum Member { + String(/* outer */ Tag, /* inner */ Tag), + Bare(Tag), +} + +impl Member { + pub(crate) fn to_expr(&self) -> hir::Expression { + match self { + Member::String(outer, inner) => hir::Expression::string(inner, outer), + Member::Bare(tag) => hir::Expression::string(tag, tag), + } + } + + pub(crate) fn tag(&self) -> Tag { + match self { + Member::String(outer, _inner) => *outer, + Member::Bare(tag) => *tag, + } + } + + pub(crate) fn to_tagged_string(&self, source: &str) -> Tagged { + match self { + Member::String(outer, inner) => inner.string(source).tagged(outer), + Member::Bare(tag) => tag.tagged_string(source), + } + } + + pub(crate) fn tagged_type_name(&self) -> Tagged<&'static str> { + match self { + Member::String(outer, _inner) => "string".tagged(outer), + Member::Bare(tag) => "word".tagged(tag), + } + } +} + +enum ColumnPathState { + Initial, + LeadingDot(Tag), + Dot(Tag, Vec, Tag), + Member(Tag, Vec), + Error(ShellError), +} + +impl ColumnPathState { + pub fn dot(self, dot: Tag) -> ColumnPathState { + match self { + ColumnPathState::Initial => ColumnPathState::LeadingDot(dot), + ColumnPathState::LeadingDot(_) => { + ColumnPathState::Error(ShellError::type_error("column", "dot".tagged(dot))) + } + ColumnPathState::Dot(..) => { + ColumnPathState::Error(ShellError::type_error("column", "dot".tagged(dot))) + } + ColumnPathState::Member(tag, members) => ColumnPathState::Dot(tag, members, dot), + ColumnPathState::Error(err) => ColumnPathState::Error(err), + } + } + + pub fn member(self, member: Member) -> ColumnPathState { + match self { + ColumnPathState::Initial => ColumnPathState::Member(member.tag(), vec![member]), + ColumnPathState::LeadingDot(tag) => { + ColumnPathState::Member(tag.until(member.tag()), vec![member]) + } + + ColumnPathState::Dot(tag, mut tags, _) => { + ColumnPathState::Member(tag.until(member.tag()), { + tags.push(member); + tags + }) + } + ColumnPathState::Member(..) => { + ColumnPathState::Error(ShellError::type_error("column", member.tagged_type_name())) + } + ColumnPathState::Error(err) => ColumnPathState::Error(err), + } + } + + pub fn into_path(self, next: Peeked) -> Result>, ShellError> { + match self { + ColumnPathState::Initial => Err(next.type_error("column path")), + ColumnPathState::LeadingDot(dot) => { + Err(ShellError::type_error("column", "dot".tagged(dot))) + } + ColumnPathState::Dot(_tag, _members, dot) => { + Err(ShellError::type_error("column", "dot".tagged(dot))) + } + ColumnPathState::Member(tag, tags) => Ok(tags.tagged(tag)), + ColumnPathState::Error(err) => Err(err), + } + } +} + +pub fn expand_column_path<'a, 'b>( + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, +) -> Result>, ShellError> { + let mut state = ColumnPathState::Initial; + + loop { + let member = MemberShape.expand_syntax(token_nodes, context); + + match member { + Err(_) => break, + Ok(member) => state = state.member(member), + } + + let dot = DotShape.expand_syntax(token_nodes, context); + + match dot { + Err(_) => break, + Ok(dot) => state = state.dot(dot), + } + } + + state.into_path(token_nodes.peek_non_ws()) +} + +#[derive(Debug, Copy, Clone)] +pub struct ColumnPathShape; + +impl ExpandSyntax for ColumnPathShape { + type Output = Tagged>; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + expand_column_path(token_nodes, context) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct MemberShape; + +impl ExpandSyntax for MemberShape { + type Output = Member; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result { + let bare = BareShape.test(token_nodes, context); + if let Some(peeked) = bare { + let node = peeked.not_eof("column")?.commit(); + return Ok(Member::Bare(node.tag())); + } + + let string = StringShape.test(token_nodes, context); + + if let Some(peeked) = string { + let node = peeked.not_eof("column")?.commit(); + let (outer, inner) = node.expect_string(); + + return Ok(Member::String(outer, inner)); + } + + Err(token_nodes.peek_any().type_error("column")) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct DotShape; + +impl SkipSyntax for DotShape { + fn skip<'a, 'b>( + &self, + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + ) -> Result<(), ShellError> { + expand_syntax(self, token_nodes, context)?; + + Ok(()) + } +} + +impl ExpandSyntax for DotShape { + type Output = Tag; + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + ) -> Result { + parse_single_node(token_nodes, "dot", |token, token_tag| { + Ok(match token { + RawToken::Operator(Operator::Dot) => token_tag, + _ => { + return Err(ShellError::type_error( + "dot", + token.type_name().tagged(token_tag), + )) + } + }) + }) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct InfixShape; + +impl ExpandSyntax for InfixShape { + type Output = (Tag, Tagged, Tag); + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let checkpoint = token_nodes.checkpoint(); + + // An infix operator must be prefixed by whitespace + let start = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?; + + // Parse the next TokenNode after the whitespace + let operator = + parse_single_node(checkpoint.iterator, "infix operator", |token, token_tag| { + Ok(match token { + // If it's an operator (and not `.`), it's a match + RawToken::Operator(operator) if operator != Operator::Dot => { + operator.tagged(token_tag) + } + + // Otherwise, it's not a match + _ => { + return Err(ShellError::type_error( + "infix operator", + token.type_name().tagged(token_tag), + )) + } + }) + })?; + + // An infix operator must be followed by whitespace + let end = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?; + + checkpoint.commit(); + + Ok((start, operator, end)) + } +} diff --git a/src/parser/hir/tokens_iterator.rs b/src/parser/hir/tokens_iterator.rs new file mode 100644 index 0000000000..c0dd9c50fd --- /dev/null +++ b/src/parser/hir/tokens_iterator.rs @@ -0,0 +1,365 @@ +pub(crate) mod debug; + +use crate::errors::ShellError; +use crate::parser::TokenNode; +use crate::{Tag, Tagged, TaggedItem}; +use derive_new::new; + +#[derive(Debug, new)] +pub struct TokensIterator<'a> { + tokens: &'a [TokenNode], + tag: Tag, + skip_ws: bool, + #[new(default)] + index: usize, + #[new(default)] + seen: indexmap::IndexSet, +} + +#[derive(Debug)] +pub struct Checkpoint<'content, 'me> { + pub(crate) iterator: &'me mut TokensIterator<'content>, + index: usize, + seen: indexmap::IndexSet, + committed: bool, +} + +impl<'content, 'me> Checkpoint<'content, 'me> { + pub(crate) fn commit(mut self) { + self.committed = true; + } +} + +impl<'content, 'me> std::ops::Drop for Checkpoint<'content, 'me> { + fn drop(&mut self) { + if !self.committed { + self.iterator.index = self.index; + self.iterator.seen = self.seen.clone(); + } + } +} + +#[derive(Debug)] +pub struct Peeked<'content, 'me> { + pub(crate) node: Option<&'content TokenNode>, + iterator: &'me mut TokensIterator<'content>, + from: usize, + to: usize, +} + +impl<'content, 'me> Peeked<'content, 'me> { + pub fn commit(&mut self) -> Option<&'content TokenNode> { + let Peeked { + node, + iterator, + from, + to, + } = self; + + let node = (*node)?; + iterator.commit(*from, *to); + Some(node) + } + + pub fn not_eof( + self, + expected: impl Into, + ) -> Result, ShellError> { + match self.node { + None => Err(ShellError::unexpected_eof( + expected, + self.iterator.eof_tag(), + )), + Some(node) => Ok(PeekedNode { + node, + iterator: self.iterator, + from: self.from, + to: self.to, + }), + } + } + + pub fn type_error(&self, expected: impl Into) -> ShellError { + peek_error(&self.node, self.iterator.eof_tag(), expected) + } +} + +#[derive(Debug)] +pub struct PeekedNode<'content, 'me> { + pub(crate) node: &'content TokenNode, + iterator: &'me mut TokensIterator<'content>, + from: usize, + to: usize, +} + +impl<'content, 'me> PeekedNode<'content, 'me> { + pub fn commit(self) -> &'content TokenNode { + let PeekedNode { + node, + iterator, + from, + to, + } = self; + + iterator.commit(from, to); + node + } + + pub fn rollback(self) {} + + pub fn type_error(&self, expected: impl Into) -> ShellError { + peek_error(&Some(self.node), self.iterator.eof_tag(), expected) + } +} + +pub fn peek_error( + node: &Option<&TokenNode>, + eof_tag: Tag, + expected: impl Into, +) -> ShellError { + match node { + None => ShellError::unexpected_eof(expected, eof_tag), + Some(node) => ShellError::type_error(expected, node.tagged_type_name()), + } +} + +impl<'content> TokensIterator<'content> { + #[cfg(test)] + pub fn all(tokens: &'content [TokenNode], tag: Tag) -> TokensIterator<'content> { + TokensIterator::new(tokens, tag, false) + } + + /// Use a checkpoint when you need to peek more than one token ahead, but can't be sure + /// that you'll succeed. + pub fn checkpoint<'me>(&'me mut self) -> Checkpoint<'content, 'me> { + let index = self.index; + let seen = self.seen.clone(); + + Checkpoint { + iterator: self, + index, + seen, + committed: false, + } + } + + pub fn anchor(&self) -> uuid::Uuid { + self.tag.anchor + } + + fn eof_tag(&self) -> Tag { + Tag::from((self.tag.span.end(), self.tag.span.end(), self.tag.anchor)) + } + + pub fn typed_tag_at_cursor(&mut self) -> Tagged<&'static str> { + let next = self.peek_any(); + + match next.node { + None => "end".tagged(self.eof_tag()), + Some(node) => node.tagged_type_name(), + } + } + + pub fn remove(&mut self, position: usize) { + self.seen.insert(position); + } + + pub fn at_end(&self) -> bool { + peek(self, self.skip_ws).is_none() + } + + pub fn at_end_possible_ws(&self) -> bool { + peek(self, true).is_none() + } + + pub fn advance(&mut self) { + self.seen.insert(self.index); + self.index += 1; + } + + pub fn extract(&mut self, f: impl Fn(&TokenNode) -> Option) -> Option<(usize, T)> { + for (i, item) in self.tokens.iter().enumerate() { + if self.seen.contains(&i) { + continue; + } + + match f(item) { + None => { + continue; + } + Some(value) => { + self.seen.insert(i); + return Some((i, value)); + } + } + } + + None + } + + pub fn move_to(&mut self, pos: usize) { + self.index = pos; + } + + pub fn restart(&mut self) { + self.index = 0; + } + + pub fn clone(&self) -> TokensIterator<'content> { + TokensIterator { + tokens: self.tokens, + tag: self.tag, + index: self.index, + seen: self.seen.clone(), + skip_ws: self.skip_ws, + } + } + + // Get the next token, not including whitespace + pub fn next_non_ws(&mut self) -> Option<&TokenNode> { + let mut peeked = start_next(self, true); + peeked.commit() + } + + // Peek the next token, not including whitespace + pub fn peek_non_ws<'me>(&'me mut self) -> Peeked<'content, 'me> { + start_next(self, true) + } + + // Peek the next token, including whitespace + pub fn peek_any<'me>(&'me mut self) -> Peeked<'content, 'me> { + start_next(self, false) + } + + fn commit(&mut self, from: usize, to: usize) { + for index in from..to { + self.seen.insert(index); + } + + self.index = to; + } + + pub fn debug_remaining(&self) -> Vec { + let mut tokens = self.clone(); + tokens.restart(); + tokens.cloned().collect() + } +} + +impl<'a> Iterator for TokensIterator<'a> { + type Item = &'a TokenNode; + + fn next(&mut self) -> Option<&'a TokenNode> { + next(self, self.skip_ws) + } +} + +fn peek<'content, 'me>( + iterator: &TokensIterator<'content>, + skip_ws: bool, +) -> Option<&'content TokenNode> { + let mut to = iterator.index; + + loop { + if to >= iterator.tokens.len() { + return None; + } + + if iterator.seen.contains(&to) { + to += 1; + continue; + } + + if to >= iterator.tokens.len() { + return None; + } + + let node = &iterator.tokens[to]; + + match node { + TokenNode::Whitespace(_) if skip_ws => { + to += 1; + } + _ => { + return Some(node); + } + } + } +} + +fn start_next<'content, 'me>( + iterator: &'me mut TokensIterator<'content>, + skip_ws: bool, +) -> Peeked<'content, 'me> { + let from = iterator.index; + let mut to = iterator.index; + + loop { + if to >= iterator.tokens.len() { + return Peeked { + node: None, + iterator, + from, + to, + }; + } + + if iterator.seen.contains(&to) { + to += 1; + continue; + } + + if to >= iterator.tokens.len() { + return Peeked { + node: None, + iterator, + from, + to, + }; + } + + let node = &iterator.tokens[to]; + + match node { + TokenNode::Whitespace(_) if skip_ws => { + to += 1; + } + _ => { + to += 1; + return Peeked { + node: Some(node), + iterator, + from, + to, + }; + } + } + } +} + +fn next<'a>(iterator: &mut TokensIterator<'a>, skip_ws: bool) -> Option<&'a TokenNode> { + loop { + if iterator.index >= iterator.tokens.len() { + return None; + } + + if iterator.seen.contains(&iterator.index) { + iterator.advance(); + continue; + } + + if iterator.index >= iterator.tokens.len() { + return None; + } + + match &iterator.tokens[iterator.index] { + TokenNode::Whitespace(_) if skip_ws => { + iterator.advance(); + } + other => { + iterator.advance(); + return Some(other); + } + } + } +} diff --git a/src/parser/hir/tokens_iterator/debug.rs b/src/parser/hir/tokens_iterator/debug.rs new file mode 100644 index 0000000000..2e26720154 --- /dev/null +++ b/src/parser/hir/tokens_iterator/debug.rs @@ -0,0 +1,30 @@ +use crate::parser::hir::tokens_iterator::TokensIterator; +use crate::traits::ToDebug; + +#[derive(Debug)] +pub(crate) enum DebugIteratorToken { + Seen(String), + Unseen(String), + Cursor, +} + +pub(crate) fn debug_tokens(iterator: &TokensIterator, source: &str) -> Vec { + let mut out = vec![]; + + for (i, token) in iterator.tokens.iter().enumerate() { + if iterator.index == i { + out.push(DebugIteratorToken::Cursor); + } + + if iterator.seen.contains(&i) { + out.push(DebugIteratorToken::Seen(format!("{}", token.debug(source)))); + } else { + out.push(DebugIteratorToken::Unseen(format!( + "{}", + token.debug(source) + ))); + } + } + + out +} diff --git a/src/parser/parse/files.rs b/src/parser/parse/files.rs index afe75ddb27..3c28237f5d 100644 --- a/src/parser/parse/files.rs +++ b/src/parser/parse/files.rs @@ -1,6 +1,7 @@ use crate::Tag; use derive_new::new; use language_reporting::{FileName, Location}; +use log::trace; use uuid::Uuid; #[derive(new, Debug, Clone)] @@ -18,7 +19,7 @@ impl language_reporting::ReportingFiles for Files { from_index: usize, to_index: usize, ) -> Option { - Some(Tag::from((from_index, to_index, file))) + Some(Tag::new(file, (from_index, to_index).into())) } fn file_id(&self, tag: Self::Span) -> Self::FileId { @@ -38,8 +39,18 @@ impl language_reporting::ReportingFiles for Files { let mut seen_lines = 0; let mut seen_bytes = 0; - for (pos, _) in source.match_indices('\n') { - if pos > byte_index { + for (pos, slice) in source.match_indices('\n') { + trace!( + "SEARCH={} SEEN={} POS={} SLICE={:?} LEN={} ALL={:?}", + byte_index, + seen_bytes, + pos, + slice, + source.len(), + source + ); + + if pos >= byte_index { return Some(language_reporting::Location::new( seen_lines, byte_index - seen_bytes, @@ -53,7 +64,7 @@ impl language_reporting::ReportingFiles for Files { if seen_lines == 0 { Some(language_reporting::Location::new(0, byte_index)) } else { - None + panic!("byte index {} wasn't valid", byte_index); } } @@ -64,7 +75,7 @@ impl language_reporting::ReportingFiles for Files { for (pos, _) in source.match_indices('\n') { if seen_lines == lineno { - return Some(Tag::from((seen_bytes, pos, file))); + return Some(Tag::new(file, (seen_bytes, pos + 1).into())); } else { seen_lines += 1; seen_bytes = pos + 1; @@ -72,16 +83,18 @@ impl language_reporting::ReportingFiles for Files { } if seen_lines == 0 { - Some(Tag::from((0, self.snippet.len() - 1, file))) + Some(Tag::new(file, (0, self.snippet.len() - 1).into())) } else { None } } fn source(&self, tag: Self::Span) -> Option { - if tag.span.start > tag.span.end { + trace!("source(tag={:?}) snippet={:?}", tag, self.snippet); + + if tag.span.start() > tag.span.end() { return None; - } else if tag.span.end >= self.snippet.len() { + } else if tag.span.end() > self.snippet.len() { return None; } Some(tag.slice(&self.snippet).to_string()) diff --git a/src/parser/parse/operator.rs b/src/parser/parse/operator.rs index 82a04ed796..7b5a5c77d8 100644 --- a/src/parser/parse/operator.rs +++ b/src/parser/parse/operator.rs @@ -11,6 +11,7 @@ pub enum Operator { GreaterThan, LessThanOrEqual, GreaterThanOrEqual, + Dot, } impl ToDebug for Operator { @@ -32,6 +33,7 @@ impl Operator { Operator::GreaterThan => ">", Operator::LessThanOrEqual => "<=", Operator::GreaterThanOrEqual => ">=", + Operator::Dot => ".", } } } @@ -52,6 +54,7 @@ impl FromStr for Operator { ">" => Ok(Operator::GreaterThan), "<=" => Ok(Operator::LessThanOrEqual), ">=" => Ok(Operator::GreaterThanOrEqual), + "." => Ok(Operator::Dot), _ => Err(()), } } diff --git a/src/parser/parse/parser.rs b/src/parser/parse/parser.rs index 33903ba37c..93ba043ba1 100644 --- a/src/parser/parse/parser.rs +++ b/src/parser/parse/parser.rs @@ -14,24 +14,54 @@ use nom::combinator::*; use nom::multi::*; use nom::sequence::*; +use derive_new::new; use log::trace; use nom::dbg; use nom::*; use nom::{AsBytes, FindSubstring, IResult, InputLength, InputTake, Slice}; use nom_locate::{position, LocatedSpanEx}; +use nom_tracable::{tracable_parser, HasTracableInfo, TracableInfo}; use serde::{Deserialize, Serialize}; use std::fmt::Debug; use std::str::FromStr; use uuid::Uuid; -pub type NomSpan<'a> = LocatedSpanEx<&'a str, Uuid>; +pub type NomSpan<'a> = LocatedSpanEx<&'a str, TracableContext>; + +#[derive(Debug, Clone, Copy, PartialEq, new)] +pub struct TracableContext { + pub(crate) origin: Uuid, + pub(crate) info: TracableInfo, +} + +impl HasTracableInfo for TracableContext { + fn get_tracable_info(&self) -> TracableInfo { + self.info + } + + fn set_tracable_info(mut self, info: TracableInfo) -> Self { + TracableContext { + origin: self.origin, + info, + } + } +} + +impl std::ops::Deref for TracableContext { + type Target = TracableInfo; + + fn deref(&self) -> &TracableInfo { + &self.info + } +} pub fn nom_input(s: &str, anchor: Uuid) -> NomSpan<'_> { - LocatedSpanEx::new_extra(s, anchor) + LocatedSpanEx::new_extra(s, TracableContext::new(anchor, TracableInfo::new())) } macro_rules! operator { ($name:tt : $token:tt ) => { + #[tracable_parser] pub fn $name(input: NomSpan) -> IResult { let start = input.offset; let (input, tag) = tag(stringify!($token))(input)?; @@ -51,25 +81,7 @@ operator! { gte: >= } operator! { lte: <= } operator! { eq: == } operator! { neq: != } - -fn trace_step<'a, T: Debug>( - input: NomSpan<'a>, - name: &str, - block: impl FnOnce(NomSpan<'a>) -> IResult, T>, -) -> IResult, T> { - trace!(target: "nu::lite_parse", "+ before {} @ {:?}", name, input); - match block(input) { - Ok((input, result)) => { - trace!(target: "nu::lite_parse", "after {} @ {:?} -> {:?}", name, input, result); - Ok((input, result)) - } - - Err(e) => { - trace!(target: "nu::lite_parse", "- failed {} :: {:?}", name, e); - Err(e) - } - } -} +operator! { dot: . } #[derive(Debug, Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize)] pub enum Number { @@ -77,6 +89,15 @@ pub enum Number { Decimal(BigDecimal), } +impl std::fmt::Display for Number { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Number::Int(int) => write!(f, "{}", int), + Number::Decimal(decimal) => write!(f, "{}", decimal), + } + } +} + macro_rules! primitive_int { ($($ty:ty)*) => { $( @@ -148,540 +169,479 @@ impl Into for BigDecimal { } } +#[tracable_parser] +pub fn number(input: NomSpan) -> IResult { + let (input, number) = raw_number(input)?; + + Ok(( + input, + TokenTreeBuilder::tagged_number(number.item, number.tag), + )) +} + +#[tracable_parser] pub fn raw_number(input: NomSpan) -> IResult> { let anchoral = input; let start = input.offset; - trace_step(input, "raw_decimal", move |input| { - let (input, neg) = opt(tag("-"))(input)?; - let (input, head) = digit1(input)?; - let dot: IResult = tag(".")(input); + let (input, neg) = opt(tag("-"))(input)?; + let (input, head) = digit1(input)?; - let input = match dot { - Ok((input, dot)) => input, + match input.fragment.chars().next() { + None => return Ok((input, RawNumber::int((start, input.offset, input.extra)))), + Some('.') => (), + Some(other) if other.is_whitespace() => { + return Ok((input, RawNumber::int((start, input.offset, input.extra)))) + } + _ => { + return Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::Tag, + ))) + } + } - // it's just an integer - Err(_) => return Ok((input, RawNumber::int((start, input.offset, input.extra)))), - }; + let dot: IResult = tag(".")(input); - let (input, tail) = digit1(input)?; + let input = match dot { + Ok((input, dot)) => input, - let end = input.offset; + // it's just an integer + Err(_) => return Ok((input, RawNumber::int((start, input.offset, input.extra)))), + }; - Ok((input, RawNumber::decimal((start, end, input.extra)))) - }) + let (input, tail) = digit1(input)?; + + let end = input.offset; + + let next = input.fragment.chars().next(); + + if let Some(next) = next { + if !next.is_whitespace() { + return Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::Tag, + ))); + } + } + + Ok((input, RawNumber::decimal((start, end, input.extra)))) } +#[tracable_parser] pub fn operator(input: NomSpan) -> IResult { - trace_step(input, "operator", |input| { - let (input, operator) = alt((gte, lte, neq, gt, lt, eq))(input)?; + let (input, operator) = alt((gte, lte, neq, gt, lt, eq))(input)?; - Ok((input, operator)) - }) + Ok((input, operator)) } +#[tracable_parser] pub fn dq_string(input: NomSpan) -> IResult { - trace_step(input, "dq_string", |input| { - let start = input.offset; - let (input, _) = char('"')(input)?; - let start1 = input.offset; - let (input, _) = many0(none_of("\""))(input)?; - let end1 = input.offset; - let (input, _) = char('"')(input)?; - let end = input.offset; - Ok(( - input, - TokenTreeBuilder::tagged_string((start1, end1, input.extra), (start, end, input.extra)), - )) - }) + let start = input.offset; + let (input, _) = char('"')(input)?; + let start1 = input.offset; + let (input, _) = many0(none_of("\""))(input)?; + let end1 = input.offset; + let (input, _) = char('"')(input)?; + let end = input.offset; + Ok(( + input, + TokenTreeBuilder::tagged_string((start1, end1, input.extra), (start, end, input.extra)), + )) } +#[tracable_parser] pub fn sq_string(input: NomSpan) -> IResult { - trace_step(input, "sq_string", move |input| { - let start = input.offset; - let (input, _) = char('\'')(input)?; - let start1 = input.offset; - let (input, _) = many0(none_of("\'"))(input)?; - let end1 = input.offset; - let (input, _) = char('\'')(input)?; - let end = input.offset; + let start = input.offset; + let (input, _) = char('\'')(input)?; + let start1 = input.offset; + let (input, _) = many0(none_of("\'"))(input)?; + let end1 = input.offset; + let (input, _) = char('\'')(input)?; + let end = input.offset; - Ok(( - input, - TokenTreeBuilder::tagged_string((start1, end1, input.extra), (start, end, input.extra)), - )) - }) + Ok(( + input, + TokenTreeBuilder::tagged_string((start1, end1, input.extra), (start, end, input.extra)), + )) } +#[tracable_parser] pub fn string(input: NomSpan) -> IResult { - trace_step(input, "string", move |input| { - alt((sq_string, dq_string))(input) - }) + alt((sq_string, dq_string))(input) } +#[tracable_parser] pub fn external(input: NomSpan) -> IResult { - trace_step(input, "external", move |input| { - let start = input.offset; - let (input, _) = tag("^")(input)?; - let (input, bare) = take_while(is_bare_char)(input)?; - let end = input.offset; + let start = input.offset; + let (input, _) = tag("^")(input)?; + let (input, bare) = take_while(is_bare_char)(input)?; + let end = input.offset; - Ok(( - input, - TokenTreeBuilder::tagged_external(bare, (start, end, input.extra)), - )) - }) + Ok(( + input, + TokenTreeBuilder::tagged_external_command(bare, (start, end, input.extra)), + )) } +#[tracable_parser] pub fn pattern(input: NomSpan) -> IResult { - trace_step(input, "bare", move |input| { - let start = input.offset; - let (input, _) = take_while1(is_start_glob_char)(input)?; - let (input, _) = take_while(is_glob_char)(input)?; + let start = input.offset; + let (input, _) = take_while1(is_start_glob_char)(input)?; + let (input, _) = take_while(is_glob_char)(input)?; - let next_char = &input.fragment.chars().nth(0); + let next_char = &input.fragment.chars().nth(0); - if let Some(next_char) = next_char { - if is_external_word_char(*next_char) { - return Err(nom::Err::Error(nom::error::make_error( - input, - nom::error::ErrorKind::TakeWhile1, - ))); - } + if let Some(next_char) = next_char { + if is_external_word_char(*next_char) { + return Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::TakeWhile1, + ))); } + } - let end = input.offset; + let end = input.offset; - Ok(( - input, - TokenTreeBuilder::tagged_pattern((start, end, input.extra)), - )) - }) + Ok(( + input, + TokenTreeBuilder::tagged_pattern((start, end, input.extra)), + )) } +#[tracable_parser] pub fn bare(input: NomSpan) -> IResult { - trace_step(input, "bare", move |input| { - let start = input.offset; - let (input, _) = take_while1(is_start_bare_char)(input)?; - let (input, _) = take_while(is_bare_char)(input)?; + let start = input.offset; + let (input, _) = take_while1(is_start_bare_char)(input)?; + let (input, last) = take_while(is_bare_char)(input)?; - let next_char = &input.fragment.chars().nth(0); + let next_char = &input.fragment.chars().nth(0); + let prev_char = last.fragment.chars().nth(0); - if let Some(next_char) = next_char { - if is_external_word_char(*next_char) || is_glob_specific_char(*next_char) { - return Err(nom::Err::Error(nom::error::make_error( - input, - nom::error::ErrorKind::TakeWhile1, - ))); - } + // if let (Some(prev), Some(next)) = (prev_char, next_char) { + // if prev == '.' && is_member_start(*next) { + // return Err(nom::Err::Error(nom::error::make_error( + // input, + // nom::error::ErrorKind::TakeWhile1, + // ))); + // } + // } + + if let Some(next_char) = next_char { + if is_external_word_char(*next_char) || is_glob_specific_char(*next_char) { + return Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::TakeWhile1, + ))); } + } - let end = input.offset; + let end = input.offset; - Ok(( - input, - TokenTreeBuilder::tagged_bare((start, end, input.extra)), - )) - }) + Ok(( + input, + TokenTreeBuilder::tagged_bare((start, end, input.extra)), + )) } +#[tracable_parser] pub fn external_word(input: NomSpan) -> IResult { - trace_step(input, "bare", move |input| { - let start = input.offset; - let (input, _) = take_while1(is_external_word_char)(input)?; - let end = input.offset; + let start = input.offset; + let (input, _) = take_while1(is_external_word_char)(input)?; + let end = input.offset; - Ok(( - input, - TokenTreeBuilder::tagged_external_word((start, end, input.extra)), - )) - }) + Ok(( + input, + TokenTreeBuilder::tagged_external_word((start, end, input.extra)), + )) } +#[tracable_parser] pub fn var(input: NomSpan) -> IResult { - trace_step(input, "var", move |input| { - let start = input.offset; - let (input, _) = tag("$")(input)?; - let (input, bare) = member(input)?; - let end = input.offset; + let start = input.offset; + let (input, _) = tag("$")(input)?; + let (input, bare) = ident(input)?; + let end = input.offset; - Ok(( - input, - TokenTreeBuilder::tagged_var(bare.tag(), (start, end, input.extra)), - )) - }) + Ok(( + input, + TokenTreeBuilder::tagged_var(bare, (start, end, input.extra)), + )) } -pub fn member(input: NomSpan) -> IResult { - trace_step(input, "identifier", move |input| { - let start = input.offset; - let (input, _) = take_while1(is_id_start)(input)?; - let (input, _) = take_while(is_id_continue)(input)?; +#[tracable_parser] +pub fn ident(input: NomSpan) -> IResult { + let start = input.offset; + let (input, _) = take_while1(is_start_bare_char)(input)?; + let (input, _) = take_while(is_bare_char)(input)?; + let end = input.offset; - let end = input.offset; - - Ok(( - input, - TokenTreeBuilder::tagged_member((start, end, input.extra)), - )) - }) + Ok((input, Tag::from((start, end, input.extra.origin)))) } +#[tracable_parser] pub fn flag(input: NomSpan) -> IResult { - trace_step(input, "flag", move |input| { - let start = input.offset; - let (input, _) = tag("--")(input)?; - let (input, bare) = bare(input)?; - let end = input.offset; + let start = input.offset; + let (input, _) = tag("--")(input)?; + let (input, bare) = bare(input)?; + let end = input.offset; - Ok(( - input, - TokenTreeBuilder::tagged_flag(bare.tag(), (start, end, input.extra)), - )) - }) + Ok(( + input, + TokenTreeBuilder::tagged_flag(bare.tag(), (start, end, input.extra)), + )) } +#[tracable_parser] pub fn shorthand(input: NomSpan) -> IResult { - trace_step(input, "shorthand", move |input| { - let start = input.offset; - let (input, _) = tag("-")(input)?; - let (input, bare) = bare(input)?; - let end = input.offset; + let start = input.offset; + let (input, _) = tag("-")(input)?; + let (input, bare) = bare(input)?; + let end = input.offset; - Ok(( - input, - TokenTreeBuilder::tagged_shorthand(bare.tag(), (start, end, input.extra)), - )) - }) -} - -pub fn raw_unit(input: NomSpan) -> IResult> { - trace_step(input, "raw_unit", move |input| { - let start = input.offset; - let (input, unit) = alt(( - tag("B"), - tag("b"), - tag("KB"), - tag("kb"), - tag("Kb"), - tag("K"), - tag("k"), - tag("MB"), - tag("mb"), - tag("Mb"), - tag("GB"), - tag("gb"), - tag("Gb"), - tag("TB"), - tag("tb"), - tag("Tb"), - tag("PB"), - tag("pb"), - tag("Pb"), - ))(input)?; - let end = input.offset; - - Ok(( - input, - Unit::from(unit.fragment).tagged((start, end, input.extra)), - )) - }) -} - -pub fn size(input: NomSpan) -> IResult { - trace_step(input, "size", move |input| { - let mut is_size = false; - let start = input.offset; - let (input, number) = raw_number(input)?; - if let Ok((input, Some(size))) = opt(raw_unit)(input) { - let end = input.offset; - - // Check to make sure there is no trailing parseable characters - if let Ok((input, Some(extra))) = opt(bare)(input) { - return Err(nom::Err::Error((input, nom::error::ErrorKind::Char))); - } - - Ok(( - input, - TokenTreeBuilder::tagged_size((number.item, *size), (start, end, input.extra)), - )) - } else { - let end = input.offset; - - // Check to make sure there is no trailing parseable characters - if let Ok((input, Some(extra))) = opt(bare)(input) { - return Err(nom::Err::Error((input, nom::error::ErrorKind::Char))); - } - - Ok(( - input, - TokenTreeBuilder::tagged_number(number.item, number.tag), - )) - } - }) + Ok(( + input, + TokenTreeBuilder::tagged_shorthand(bare.tag(), (start, end, input.extra)), + )) } +#[tracable_parser] pub fn leaf(input: NomSpan) -> IResult { - trace_step(input, "leaf", move |input| { - let (input, node) = alt(( - size, - string, - operator, - flag, - shorthand, - var, - external, - bare, - pattern, - external_word, - ))(input)?; + let (input, node) = alt((number, string, operator, flag, shorthand, var, external))(input)?; - Ok((input, node)) - }) + Ok((input, node)) } -pub fn token_list(input: NomSpan) -> IResult> { - trace_step(input, "token_list", move |input| { - let (input, first) = node(input)?; - let (input, list) = many0(pair(space1, node))(input)?; +#[tracable_parser] +pub fn token_list(input: NomSpan) -> IResult>> { + let start = input.offset; + let (input, first) = node(input)?; - Ok((input, make_token_list(None, first, list, None))) - }) + let (input, mut list) = many0(pair(alt((whitespace, dot)), node))(input)?; + + let end = input.offset; + + Ok(( + input, + make_token_list(first, list, None).tagged((start, end, input.extra.origin)), + )) } -pub fn spaced_token_list(input: NomSpan) -> IResult> { - trace_step(input, "spaced_token_list", move |input| { - let (input, sp_left) = opt(space1)(input)?; - let (input, first) = node(input)?; - let (input, list) = many0(pair(space1, node))(input)?; - let (input, sp_right) = opt(space1)(input)?; +#[tracable_parser] +pub fn spaced_token_list(input: NomSpan) -> IResult>> { + let start = input.offset; + let (input, pre_ws) = opt(whitespace)(input)?; + let (input, items) = token_list(input)?; + let (input, post_ws) = opt(whitespace)(input)?; + let end = input.offset; - Ok((input, make_token_list(sp_left, first, list, sp_right))) - }) + let mut out = vec![]; + + out.extend(pre_ws); + out.extend(items.item); + out.extend(post_ws); + + Ok((input, out.tagged((start, end, input.extra.origin)))) } fn make_token_list( - sp_left: Option, - first: TokenNode, - list: Vec<(NomSpan, TokenNode)>, - sp_right: Option, + first: Vec, + list: Vec<(TokenNode, Vec)>, + sp_right: Option, ) -> Vec { let mut nodes = vec![]; - if let Some(sp_left) = sp_left { - nodes.push(TokenNode::Whitespace(Tag::from(sp_left))); - } + nodes.extend(first); - nodes.push(first); - - for (ws, token) in list { - nodes.push(TokenNode::Whitespace(Tag::from(ws))); - nodes.push(token); + for (left, right) in list { + nodes.push(left); + nodes.extend(right); } if let Some(sp_right) = sp_right { - nodes.push(TokenNode::Whitespace(Tag::from(sp_right))); + nodes.push(sp_right); } nodes } +#[tracable_parser] pub fn whitespace(input: NomSpan) -> IResult { - trace_step(input, "whitespace", move |input| { - let left = input.offset; - let (input, ws1) = space1(input)?; - let right = input.offset; + let left = input.offset; + let (input, ws1) = space1(input)?; + let right = input.offset; - Ok(( - input, - TokenTreeBuilder::tagged_ws((left, right, input.extra)), - )) - }) -} - -pub fn delimited_paren(input: NomSpan) -> IResult { - trace_step(input, "delimited_paren", move |input| { - let left = input.offset; - let (input, _) = char('(')(input)?; - let (input, ws1) = opt(whitespace)(input)?; - let (input, inner_items) = opt(token_list)(input)?; - let (input, ws2) = opt(whitespace)(input)?; - let (input, _) = char(')')(input)?; - let right = input.offset; - - let mut items = vec![]; - - if let Some(space) = ws1 { - items.push(space); - } - - if let Some(inner_items) = inner_items { - items.extend(inner_items); - } - - if let Some(space) = ws2 { - items.push(space); - } - - Ok(( - input, - TokenTreeBuilder::tagged_parens(items, (left, right, input.extra)), - )) - }) -} - -pub fn delimited_square(input: NomSpan) -> IResult { - trace_step(input, "delimited_paren", move |input| { - let left = input.offset; - let (input, _) = char('[')(input)?; - let (input, ws1) = opt(whitespace)(input)?; - let (input, inner_items) = opt(token_list)(input)?; - let (input, ws2) = opt(whitespace)(input)?; - let (input, _) = char(']')(input)?; - let right = input.offset; - - let mut items = vec![]; - - if let Some(space) = ws1 { - items.push(space); - } - - if let Some(inner_items) = inner_items { - items.extend(inner_items); - } - - if let Some(space) = ws2 { - items.push(space); - } - - Ok(( - input, - TokenTreeBuilder::tagged_square(items, (left, right, input.extra)), - )) - }) -} - -pub fn delimited_brace(input: NomSpan) -> IResult { - trace_step(input, "delimited_brace", move |input| { - let left = input.offset; - let (input, _) = char('{')(input)?; - let (input, _) = opt(space1)(input)?; - let (input, items) = opt(token_list)(input)?; - let (input, _) = opt(space1)(input)?; - let (input, _) = char('}')(input)?; - let right = input.offset; - - Ok(( - input, - TokenTreeBuilder::tagged_brace( - items.unwrap_or_else(|| vec![]), - (left, right, input.extra), - ), - )) - }) -} - -pub fn raw_call(input: NomSpan) -> IResult> { - trace_step(input, "raw_call", move |input| { - let left = input.offset; - let (input, items) = token_list(input)?; - let right = input.offset; - - Ok(( - input, - TokenTreeBuilder::tagged_call(items, (left, right, input.extra)), - )) - }) -} - -pub fn path(input: NomSpan) -> IResult { - trace_step(input, "path", move |input| { - let left = input.offset; - let (input, head) = node1(input)?; - let (input, _) = tag(".")(input)?; - let (input, tail) = separated_list(tag("."), alt((member, string)))(input)?; - let right = input.offset; - - Ok(( - input, - TokenTreeBuilder::tagged_path((head, tail), (left, right, input.extra)), - )) - }) -} - -pub fn node1(input: NomSpan) -> IResult { - trace_step(input, "node1", alt((leaf, delimited_paren))) -} - -pub fn node(input: NomSpan) -> IResult { - trace_step( + Ok(( input, - "node", - alt(( - path, - leaf, - delimited_paren, - delimited_brace, - delimited_square, - )), - ) + TokenTreeBuilder::tagged_ws((left, right, input.extra)), + )) } +pub fn delimited(input: NomSpan, delimiter: Delimiter) -> IResult>> { + let left = input.offset; + let (input, _) = char(delimiter.open())(input)?; + let (input, inner_items) = opt(spaced_token_list)(input)?; + let (input, _) = char(delimiter.close())(input)?; + let right = input.offset; + + let mut items = vec![]; + + if let Some(inner_items) = inner_items { + items.extend(inner_items.item); + } + + Ok((input, items.tagged((left, right, input.extra.origin)))) +} + +#[tracable_parser] +pub fn delimited_paren(input: NomSpan) -> IResult { + let (input, tokens) = delimited(input, Delimiter::Paren)?; + + Ok(( + input, + TokenTreeBuilder::tagged_parens(tokens.item, tokens.tag), + )) +} + +#[tracable_parser] +pub fn delimited_square(input: NomSpan) -> IResult { + let (input, tokens) = delimited(input, Delimiter::Square)?; + + Ok(( + input, + TokenTreeBuilder::tagged_square(tokens.item, tokens.tag), + )) +} + +#[tracable_parser] +pub fn delimited_brace(input: NomSpan) -> IResult { + let (input, tokens) = delimited(input, Delimiter::Brace)?; + + Ok(( + input, + TokenTreeBuilder::tagged_brace(tokens.item, tokens.tag), + )) +} + +#[tracable_parser] +pub fn raw_call(input: NomSpan) -> IResult> { + let left = input.offset; + let (input, items) = token_list(input)?; + let right = input.offset; + + Ok(( + input, + TokenTreeBuilder::tagged_call(items.item, (left, right, input.extra)), + )) +} + +#[tracable_parser] +pub fn bare_path(input: NomSpan) -> IResult> { + let (input, head) = alt((bare, dot))(input)?; + + let (input, tail) = many0(alt((bare, dot, string)))(input)?; + + let next_char = &input.fragment.chars().nth(0); + + if is_boundary(*next_char) { + let mut result = vec![head]; + result.extend(tail); + + Ok((input, result)) + } else { + Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::Many0, + ))) + } +} + +#[tracable_parser] +pub fn pattern_path(input: NomSpan) -> IResult> { + let (input, head) = alt((pattern, dot))(input)?; + + let (input, tail) = many0(alt((pattern, dot, string)))(input)?; + + let next_char = &input.fragment.chars().nth(0); + + if is_boundary(*next_char) { + let mut result = vec![head]; + result.extend(tail); + + Ok((input, result)) + } else { + Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::Many0, + ))) + } +} + +#[tracable_parser] +pub fn node1(input: NomSpan) -> IResult { + alt((leaf, bare, pattern, external_word, delimited_paren))(input) +} + +#[tracable_parser] +pub fn node(input: NomSpan) -> IResult> { + alt(( + to_list(leaf), + bare_path, + pattern_path, + to_list(external_word), + to_list(delimited_paren), + to_list(delimited_brace), + to_list(delimited_square), + ))(input) +} + +fn to_list( + parser: impl Fn(NomSpan) -> IResult, +) -> impl Fn(NomSpan) -> IResult> { + move |input| { + let (input, next) = parser(input)?; + + Ok((input, vec![next])) + } +} + +#[tracable_parser] +pub fn nodes(input: NomSpan) -> IResult { + let (input, tokens) = token_list(input)?; + + Ok(( + input, + TokenTreeBuilder::tagged_token_list(tokens.item, tokens.tag), + )) +} + +#[tracable_parser] pub fn pipeline(input: NomSpan) -> IResult { - trace_step(input, "pipeline", |input| { - let start = input.offset; - let (input, head) = opt(tuple((opt(space1), raw_call, opt(space1))))(input)?; - let (input, items) = trace_step( + let start = input.offset; + let (input, head) = spaced_token_list(input)?; + let (input, items) = many0(tuple((tag("|"), spaced_token_list)))(input)?; + + if input.input_len() != 0 { + return Err(Err::Error(error_position!( input, - "many0", - many0(tuple((tag("|"), opt(space1), raw_call, opt(space1)))), - )?; - - let (input, tail) = opt(space1)(input)?; - let (input, newline) = opt(multispace1)(input)?; - - if input.input_len() != 0 { - return Err(Err::Error(error_position!( - input, - nom::error::ErrorKind::Eof - ))); - } - - let end = input.offset; - - Ok(( - input, - TokenTreeBuilder::tagged_pipeline( - (make_call_list(head, items), tail.map(Tag::from)), - (start, end, input.extra), - ), - )) - }) -} - -fn make_call_list( - head: Option<(Option, Tagged, Option)>, - items: Vec<(NomSpan, Option, Tagged, Option)>, -) -> Vec { - let mut out = vec![]; - - if let Some(head) = head { - let el = PipelineElement::new(None, head.0.map(Tag::from), head.1, head.2.map(Tag::from)); - - out.push(el); + nom::error::ErrorKind::Eof + ))); } - for (pipe, ws1, call, ws2) in items { - let el = PipelineElement::new( - Some(pipe).map(Tag::from), - ws1.map(Tag::from), - call, - ws2.map(Tag::from), - ); + let end = input.offset; - out.push(el); - } + let head_tag = head.tag(); + let mut all_items: Vec> = + vec![PipelineElement::new(None, head).tagged(head_tag)]; - out + all_items.extend(items.into_iter().map(|(pipe, items)| { + let items_tag = items.tag(); + PipelineElement::new(Some(Tag::from(pipe)), items).tagged(Tag::from(pipe).until(items_tag)) + })); + + Ok(( + input, + TokenTreeBuilder::tagged_pipeline(all_items, (start, end, input.extra)), + )) } fn int(frag: &str, neg: Option) -> i64 { @@ -693,9 +653,19 @@ fn int(frag: &str, neg: Option) -> i64 { } } +fn is_boundary(c: Option) -> bool { + match c { + None => true, + Some(')') | Some(']') | Some('}') => true, + Some(c) if c.is_whitespace() => true, + _ => false, + } +} + fn is_external_word_char(c: char) -> bool { match c { - ';' | '|' | '#' | '-' | '"' | '\'' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '`' => false, + ';' | '|' | '#' | '-' | '"' | '\'' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '`' + | '.' => false, other if other.is_whitespace() => false, _ => true, } @@ -717,8 +687,7 @@ fn is_glob_char(c: char) -> bool { fn is_start_bare_char(c: char) -> bool { match c { '+' => false, - _ if c.is_alphabetic() => true, - '.' => true, + _ if c.is_alphanumeric() => true, '\\' => true, '/' => true, '_' => true, @@ -732,7 +701,6 @@ fn is_bare_char(c: char) -> bool { match c { '+' => false, _ if c.is_alphanumeric() => true, - '.' => true, '\\' => true, '/' => true, '_' => true, @@ -759,6 +727,16 @@ fn is_id_continue(c: char) -> bool { } } +fn is_member_start(c: char) -> bool { + match c { + '"' | '\'' => true, + '1'..='9' => true, + + other if is_id_start(other) => true, + _ => false, + } +} + #[cfg(test)] mod tests { use super::*; @@ -768,41 +746,6 @@ mod tests { pub type CurriedNode = Box T + 'static>; - macro_rules! assert_leaf { - (parsers [ $($name:tt)* ] $input:tt -> $left:tt .. $right:tt { $kind:tt $parens:tt } ) => { - $( - assert_eq!( - apply($name, stringify!($name), $input), - token(RawToken::$kind $parens, $left, $right) - ); - )* - - assert_eq!( - apply(leaf, "leaf", $input), - token(RawToken::$kind $parens, $left, $right) - ); - - assert_eq!( - apply(leaf, "leaf", $input), - token(RawToken::$kind $parens, $left, $right) - ); - - assert_eq!( - apply(node, "node", $input), - token(RawToken::$kind $parens, $left, $right) - ); - }; - - (parsers [ $($name:tt)* ] $input:tt -> $left:tt .. $right:tt { $kind:tt } ) => { - $( - assert_eq!( - apply($name, stringify!($name), $input), - token(RawToken::$kind, $left, $right) - ); - )* - } - } - macro_rules! equal_tokens { ($source:tt -> $tokens:expr) => { let result = apply(pipeline, "pipeline", $source); @@ -823,53 +766,50 @@ mod tests { assert_eq!(debug_result, debug_expected) } } - - // apply(pipeline, "pipeline", r#"cargo +nightly run"#), - // build_token(b::pipeline(vec![( - // None, - // b::call( - // b::bare("cargo"), - // vec![ - // b::sp(), - // b::external_word("+nightly"), - // b::sp(), - // b::bare("run") - // ] - // ), - // None - // )])) }; + + (<$parser:tt> $source:tt -> $tokens:expr) => { + let result = apply($parser, stringify!($parser), $source); + let (expected_tree, expected_source) = TokenTreeBuilder::build(uuid::Uuid::nil(), $tokens); + + if result != expected_tree { + let debug_result = format!("{}", result.debug($source)); + let debug_expected = format!("{}", expected_tree.debug(&expected_source)); + + if debug_result == debug_expected { + assert_eq!( + result, expected_tree, + "NOTE: actual and expected had equivalent debug serializations, source={:?}, debug_expected={:?}", + $source, + debug_expected + ) + } else { + assert_eq!(debug_result, debug_expected) + } + } + }; + } #[test] fn test_integer() { - assert_leaf! { - parsers [ size ] - "123" -> 0..3 { Number(RawNumber::int((0, 3, test_uuid())).item) } + equal_tokens! { + + "123" -> b::token_list(vec![b::int(123)]) } - assert_leaf! { - parsers [ size ] - "-123" -> 0..4 { Number(RawNumber::int((0, 4, test_uuid())).item) } - } - } - - #[test] - fn test_size() { - assert_leaf! { - parsers [ size ] - "123MB" -> 0..5 { Size(RawNumber::int((0, 3, test_uuid())).item, Unit::MB) } - } - - assert_leaf! { - parsers [ size ] - "10GB" -> 0..4 { Size(RawNumber::int((0, 2, test_uuid())).item, Unit::GB) } + equal_tokens! { + + "-123" -> b::token_list(vec![b::int(-123)]) } } #[test] fn test_operator() { - assert_eq!(apply(node, "node", ">"), build_token(b::op(">"))); + equal_tokens! { + + ">" -> b::token_list(vec![b::op(">")]) + } // assert_leaf! { // parsers [ operator ] @@ -899,37 +839,50 @@ mod tests { #[test] fn test_string() { - assert_leaf! { - parsers [ string dq_string ] - r#""hello world""# -> 0..13 { String(tag(1, 12)) } + equal_tokens! { + + r#""hello world""# -> b::token_list(vec![b::string("hello world")]) } - assert_leaf! { - parsers [ string sq_string ] - r"'hello world'" -> 0..13 { String(tag(1, 12)) } + equal_tokens! { + + r#"'hello world'"# -> b::token_list(vec![b::string("hello world")]) } } #[test] fn test_bare() { - assert_leaf! { - parsers [ bare ] - "hello" -> 0..5 { Bare } + equal_tokens! { + + "hello" -> b::token_list(vec![b::bare("hello")]) + } + } + + #[test] + fn test_simple_path() { + equal_tokens! { + + "450MB" -> b::token_list(vec![b::bare("450MB")]) } - assert_leaf! { - parsers [ bare ] - "chrome.exe" -> 0..10 { Bare } + equal_tokens! { + + "chrome.exe" -> b::token_list(vec![b::bare("chrome"), b::op(Operator::Dot), b::bare("exe")]) } - assert_leaf! { - parsers [ bare ] - r"C:\windows\system.dll" -> 0..21 { Bare } + equal_tokens! { + + ".azure" -> b::token_list(vec![b::op(Operator::Dot), b::bare("azure")]) } - assert_leaf! { - parsers [ bare ] - r"C:\Code\-testing\my_tests.js" -> 0..28 { Bare } + equal_tokens! { + + r"C:\windows\system.dll" -> b::token_list(vec![b::bare(r"C:\windows\system"), b::op(Operator::Dot), b::bare("dll")]) + } + + equal_tokens! { + + r"C:\Code\-testing\my_tests.js" -> b::token_list(vec![b::bare(r"C:\Code\-testing\my_tests"), b::op(Operator::Dot), b::bare("js")]) } } @@ -956,223 +909,170 @@ mod tests { #[test] fn test_variable() { - assert_leaf! { - parsers [ var ] - "$it" -> 0..3 { Variable(tag(1, 3)) } + equal_tokens! { + + "$it" -> b::token_list(vec![b::var("it")]) } - assert_leaf! { - parsers [ var ] - "$name" -> 0..5 { Variable(tag(1, 5)) } + equal_tokens! { + + "$name" -> b::token_list(vec![b::var("name")]) } } #[test] fn test_external() { - assert_leaf! { - parsers [ external ] - "^ls" -> 0..3 { ExternalCommand(tag(1, 3)) } + equal_tokens! { + + "^ls" -> b::token_list(vec![b::external_command("ls")]) + } + } + + #[test] + fn test_dot_prefixed_name() { + equal_tokens! { + + ".azure" -> b::token_list(vec![b::op("."), b::bare("azure")]) } } #[test] fn test_delimited_paren() { - assert_eq!( - apply(node, "node", "(abc)"), - build_token(b::parens(vec![b::bare("abc")])) - ); + equal_tokens! { + + "(abc)" -> b::token_list(vec![b::parens(vec![b::bare("abc")])]) + } - assert_eq!( - apply(node, "node", "( abc )"), - build_token(b::parens(vec![b::ws(" "), b::bare("abc"), b::ws(" ")])) - ); + equal_tokens! { + + "( abc )" -> b::token_list(vec![b::parens(vec![b::ws(" "), b::bare("abc"), b::ws(" ")])]) + } - assert_eq!( - apply(node, "node", "( abc def )"), - build_token(b::parens(vec![ - b::ws(" "), - b::bare("abc"), - b::sp(), - b::bare("def"), - b::sp() - ])) - ); + equal_tokens! { + + "( abc def )" -> b::token_list(vec![b::parens(vec![b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" ")])]) + } - assert_eq!( - apply(node, "node", "( abc def 123 456GB )"), - build_token(b::parens(vec![ - b::ws(" "), - b::bare("abc"), - b::sp(), - b::bare("def"), - b::sp(), - b::int(123), - b::sp(), - b::size(456, "GB"), - b::sp() - ])) - ); + equal_tokens! { + + "( abc def 123 456GB )" -> b::token_list(vec![b::parens(vec![ + b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" "), b::int(123), b::ws(" "), b::bare("456GB"), b::ws(" ") + ])]) + } } #[test] fn test_delimited_square() { - assert_eq!( - apply(node, "node", "[abc]"), - build_token(b::square(vec![b::bare("abc")])) - ); + equal_tokens! { + + "[abc]" -> b::token_list(vec![b::square(vec![b::bare("abc")])]) + } - assert_eq!( - apply(node, "node", "[ abc ]"), - build_token(b::square(vec![b::ws(" "), b::bare("abc"), b::ws(" ")])) - ); + equal_tokens! { + + "[ abc ]" -> b::token_list(vec![b::square(vec![b::ws(" "), b::bare("abc"), b::ws(" ")])]) + } - assert_eq!( - apply(node, "node", "[ abc def ]"), - build_token(b::square(vec![ - b::ws(" "), - b::bare("abc"), - b::sp(), - b::bare("def"), - b::sp() - ])) - ); + equal_tokens! { + + "[ abc def ]" -> b::token_list(vec![b::square(vec![b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" ")])]) + } - assert_eq!( - apply(node, "node", "[ abc def 123 456GB ]"), - build_token(b::square(vec![ - b::ws(" "), - b::bare("abc"), - b::sp(), - b::bare("def"), - b::sp(), - b::int(123), - b::sp(), - b::size(456, "GB"), - b::sp() - ])) - ); + equal_tokens! { + + "[ abc def 123 456GB ]" -> b::token_list(vec![b::square(vec![ + b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" "), b::int(123), b::ws(" "), b::bare("456GB"), b::ws(" ") + ])]) + } } #[test] fn test_path() { let _ = pretty_env_logger::try_init(); - assert_eq!( - apply(node, "node", "$it.print"), - build_token(b::path(b::var("it"), vec![b::member("print")])) - ); - assert_eq!( - apply(node, "node", "$head.part1.part2"), - build_token(b::path( - b::var("head"), - vec![b::member("part1"), b::member("part2")] - )) - ); + equal_tokens! { + + "$it.print" -> b::token_list(vec![b::var("it"), b::op("."), b::bare("print")]) + } - assert_eq!( - apply(node, "node", "( hello ).world"), - build_token(b::path( - b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), - vec![b::member("world")] - )) - ); + equal_tokens! { + + "$head.part1.part2" -> b::token_list(vec![b::var("head"), b::op("."), b::bare("part1"), b::op("."), b::bare("part2")]) + } - assert_eq!( - apply(node, "node", "( hello ).\"world\""), - build_token(b::path( - b::parens(vec![b::sp(), b::bare("hello"), b::sp()],), - vec![b::string("world")] - )) - ); + equal_tokens! { + + "( hello ).world" -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::op("."), b::bare("world")]) + } + + equal_tokens! { + + r#"( hello )."world""# -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::op("."), b::string("world")]) + } } #[test] fn test_nested_path() { - assert_eq!( - apply( - node, - "node", - "( $it.is.\"great news\".right yep $yep ).\"world\"" - ), - build_token(b::path( - b::parens(vec![ - b::sp(), - b::path( + equal_tokens! { + + r#"( $it.is."great news".right yep $yep )."world""# -> b::token_list( + vec![ + b::parens(vec![ + b::sp(), b::var("it"), - vec![b::member("is"), b::string("great news"), b::member("right")] - ), - b::sp(), - b::bare("yep"), - b::sp(), - b::var("yep"), - b::sp() - ]), - vec![b::string("world")] - )) - ) + b::op("."), + b::bare("is"), + b::op("."), + b::string("great news"), + b::op("."), + b::bare("right"), + b::sp(), + b::bare("yep"), + b::sp(), + b::var("yep"), + b::sp() + ]), + b::op("."), b::string("world")] + ) + } } #[test] fn test_smoke_single_command() { - assert_eq!( - apply(raw_call, "raw_call", "git add ."), - build(b::call( - b::bare("git"), - vec![b::sp(), b::bare("add"), b::sp(), b::bare(".")] - )) - ); + equal_tokens! { + + "git add ." -> b::token_list(vec![b::bare("git"), b::sp(), b::bare("add"), b::sp(), b::op(".")]) + } - assert_eq!( - apply(raw_call, "raw_call", "open Cargo.toml"), - build(b::call( - b::bare("open"), - vec![b::sp(), b::bare("Cargo.toml")] - )) - ); + equal_tokens! { + + "open Cargo.toml" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::op("."), b::bare("toml")]) + } - assert_eq!( - apply(raw_call, "raw_call", "select package.version"), - build(b::call( - b::bare("select"), - vec![b::sp(), b::bare("package.version")] - )) - ); + equal_tokens! { + + "select package.version" -> b::token_list(vec![b::bare("select"), b::sp(), b::bare("package"), b::op("."), b::bare("version")]) + } - assert_eq!( - apply(raw_call, "raw_call", "echo $it"), - build(b::call(b::bare("echo"), vec![b::sp(), b::var("it")])) - ); + equal_tokens! { + + "echo $it" -> b::token_list(vec![b::bare("echo"), b::sp(), b::var("it")]) + } - assert_eq!( - apply(raw_call, "raw_call", "open Cargo.toml --raw"), - build(b::call( - b::bare("open"), - vec![b::sp(), b::bare("Cargo.toml"), b::sp(), b::flag("raw")] - )) - ); + equal_tokens! { + + "open Cargo.toml --raw" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::op("."), b::bare("toml"), b::sp(), b::flag("raw")]) + } - assert_eq!( - apply(raw_call, "raw_call", "open Cargo.toml -r"), - build(b::call( - b::bare("open"), - vec![b::sp(), b::bare("Cargo.toml"), b::sp(), b::shorthand("r")] - )) - ); + equal_tokens! { + + "open Cargo.toml -r" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::op("."), b::bare("toml"), b::sp(), b::shorthand("r")]) + } - assert_eq!( - apply(raw_call, "raw_call", "config --set tabs 2"), - build(b::call( - b::bare("config"), - vec![ - b::sp(), - b::flag("set"), - b::sp(), - b::bare("tabs"), - b::sp(), - b::int(2) - ] - )) - ); + equal_tokens! { + + "config --set tabs 2" -> b::token_list(vec![b::bare("config"), b::sp(), b::flag("set"), b::sp(), b::bare("tabs"), b::sp(), b::int(2)]) + } } #[test] @@ -1181,120 +1081,159 @@ mod tests { equal_tokens!( "cargo +nightly run" -> - b::pipeline(vec![( - None, - b::call( - b::bare("cargo"), - vec![ - b::sp(), - b::external_word("+nightly"), - b::sp(), - b::bare("run") - ] - ), - None - )]) + b::pipeline(vec![vec![ + b::bare("cargo"), + b::sp(), + b::external_word("+nightly"), + b::sp(), + b::bare("run") + ]]) ); equal_tokens!( "rm foo%bar" -> - b::pipeline(vec![( - None, - b::call(b::bare("rm"), vec![b::sp(), b::external_word("foo%bar"),]), - None - )]) + b::pipeline(vec![vec![ + b::bare("rm"), b::sp(), b::external_word("foo%bar") + ]]) ); equal_tokens!( "rm foo%bar" -> - b::pipeline(vec![( - None, - b::call(b::bare("rm"), vec![b::sp(), b::external_word("foo%bar"),]), - None - )]) + b::pipeline(vec![vec![ + b::bare("rm"), b::sp(), b::external_word("foo%bar"), + ]]) ); } #[test] - fn test_smoke_pipeline() { + fn test_pipeline() { let _ = pretty_env_logger::try_init(); - assert_eq!( - apply( - pipeline, - "pipeline", - r#"git branch --merged | split-row "`n" | where $it != "* master""# - ), - build_token(b::pipeline(vec![ - ( - None, - b::call( - b::bare("git"), - vec![b::sp(), b::bare("branch"), b::sp(), b::flag("merged")] - ), - Some(" ") - ), - ( - Some(" "), - b::call(b::bare("split-row"), vec![b::sp(), b::string("`n")]), - Some(" ") - ), - ( - Some(" "), - b::call( - b::bare("where"), - vec![ - b::sp(), - b::var("it"), - b::sp(), - b::op("!="), - b::sp(), - b::string("* master") - ] - ), - None - ) - ])) - ); - - assert_eq!( - apply(pipeline, "pipeline", "ls | where { $it.size > 100 }"), - build_token(b::pipeline(vec![ - (None, b::call(b::bare("ls"), vec![]), Some(" ")), - ( - Some(" "), - b::call( - b::bare("where"), - vec![ - b::sp(), - b::braced(vec![ - b::path(b::var("it"), vec![b::member("size")]), - b::sp(), - b::op(">"), - b::sp(), - b::int(100) - ]) - ] - ), - None - ) - ])) - ) + equal_tokens! { + "sys | echo" -> b::pipeline(vec![ + vec![ + b::bare("sys"), b::sp() + ], + vec![ + b::sp(), b::bare("echo") + ] + ]) + } } - fn apply( - f: impl Fn(NomSpan) -> Result<(NomSpan, T), nom::Err<(NomSpan, nom::error::ErrorKind)>>, + #[test] + fn test_patterns() { + equal_tokens! { + + "cp ../formats/*" -> b::pipeline(vec![vec![b::bare("cp"), b::ws(" "), b::op("."), b::op("."), b::pattern("/formats/*")]]) + } + + equal_tokens! { + + "cp * /dev/null" -> b::pipeline(vec![vec![b::bare("cp"), b::ws(" "), b::pattern("*"), b::ws(" "), b::bare("/dev/null")]]) + } + } + + // #[test] + // fn test_pseudo_paths() { + // let _ = pretty_env_logger::try_init(); + + // equal_tokens!( + // r#"sys | where cpu."max ghz" > 1"# -> + // b::pipeline(vec![ + // (None, b::call(b::bare("sys"), vec![]), Some(" ")), + // ( + // Some(" "), + // b::call( + // b::bare("where"), + // vec![ + // b::sp(), + // b::path(b::bare("cpu"), vec![b::string("max ghz")]), + // b::sp(), + // b::op(">"), + // b::sp(), + // b::int(1) + // ] + // ), + // None + // ) + // ]) + // ); + // } + + // #[test] + // fn test_smoke_pipeline() { + // let _ = pretty_env_logger::try_init(); + + // assert_eq!( + // apply( + // pipeline, + // "pipeline", + // r#"git branch --merged | split-row "`n" | where $it != "* master""# + // ), + // build_token(b::pipeline(vec![ + // ( + // None, + // b::call( + // b::bare("git"), + // vec![b::sp(), b::bare("branch"), b::sp(), b::flag("merged")] + // ), + // Some(" ") + // ), + // ( + // Some(" "), + // b::call(b::bare("split-row"), vec![b::sp(), b::string("`n")]), + // Some(" ") + // ), + // ( + // Some(" "), + // b::call( + // b::bare("where"), + // vec![ + // b::sp(), + // b::var("it"), + // b::sp(), + // b::op("!="), + // b::sp(), + // b::string("* master") + // ] + // ), + // None + // ) + // ])) + // ); + + // assert_eq!( + // apply(pipeline, "pipeline", "ls | where { $it.size > 100 }"), + // build_token(b::pipeline(vec![ + // (None, b::call(b::bare("ls"), vec![]), Some(" ")), + // ( + // Some(" "), + // b::call( + // b::bare("where"), + // vec![ + // b::sp(), + // b::braced(vec![ + // b::path(b::var("it"), vec![b::member("size")]), + // b::sp(), + // b::op(">"), + // b::sp(), + // b::int(100) + // ]) + // ] + // ), + // None + // ) + // ])) + // ) + // } + + fn apply( + f: impl Fn(NomSpan) -> Result<(NomSpan, TokenNode), nom::Err<(NomSpan, nom::error::ErrorKind)>>, desc: &str, string: &str, - ) -> T { - match f(NomSpan::new_extra(string, uuid::Uuid::nil())) { - Ok(v) => v.1, - Err(other) => { - println!("{:?}", other); - println!("for {} @ {}", string, desc); - panic!("No dice"); - } - } + ) -> TokenNode { + f(nom_input(string, uuid::Uuid::nil())).unwrap().1 } fn tag(left: usize, right: usize) -> Tag { @@ -1312,17 +1251,6 @@ mod tests { TokenNode::Delimited(spanned) } - fn path(head: TokenNode, tail: Vec, left: usize, right: usize) -> TokenNode { - let tag = head.tag(); - - let node = PathNode::new( - Box::new(head), - tail.into_iter().map(TokenNode::Token).collect(), - ); - let spanned = node.tagged((left, right, tag.anchor)); - TokenNode::Path(spanned) - } - fn token(token: RawToken, left: usize, right: usize) -> TokenNode { TokenNode::Token(token.tagged((left, right, uuid::Uuid::nil()))) } diff --git a/src/parser/parse/pipeline.rs b/src/parser/parse/pipeline.rs index 42bbe23a18..36813e39c4 100644 --- a/src/parser/parse/pipeline.rs +++ b/src/parser/parse/pipeline.rs @@ -1,4 +1,4 @@ -use crate::parser::CallNode; +use crate::parser::TokenNode; use crate::traits::ToDebug; use crate::{Tag, Tagged}; use derive_new::new; @@ -7,20 +7,16 @@ use std::fmt; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)] pub struct Pipeline { - pub(crate) parts: Vec, - pub(crate) post_ws: Option, + pub(crate) parts: Vec>, + // pub(crate) post_ws: Option, } impl ToDebug for Pipeline { fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { - for part in &self.parts { + for part in self.parts.iter() { write!(f, "{}", part.debug(source))?; } - if let Some(post_ws) = self.post_ws { - write!(f, "{}", post_ws.slice(source))? - } - Ok(()) } } @@ -28,10 +24,7 @@ impl ToDebug for Pipeline { #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] pub struct PipelineElement { pub pipe: Option, - pub pre_ws: Option, - #[get = "pub(crate)"] - call: Tagged, - pub post_ws: Option, + pub tokens: Tagged>, } impl ToDebug for PipelineElement { @@ -40,14 +33,8 @@ impl ToDebug for PipelineElement { write!(f, "{}", pipe.slice(source))?; } - if let Some(pre_ws) = self.pre_ws { - write!(f, "{}", pre_ws.slice(source))?; - } - - write!(f, "{}", self.call.debug(source))?; - - if let Some(post_ws) = self.post_ws { - write!(f, "{}", post_ws.slice(source))?; + for token in &self.tokens.item { + write!(f, "{}", token.debug(source))?; } Ok(()) diff --git a/src/parser/parse/token_tree.rs b/src/parser/parse/token_tree.rs index e0072360e8..8cbb28264b 100644 --- a/src/parser/parse/token_tree.rs +++ b/src/parser/parse/token_tree.rs @@ -1,5 +1,6 @@ use crate::errors::ShellError; -use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*}; +use crate::parser::parse::{call_node::*, flag::*, pipeline::*, tokens::*}; +use crate::prelude::*; use crate::traits::ToDebug; use crate::{Tag, Tagged, Text}; use derive_new::new; @@ -12,15 +13,14 @@ pub enum TokenNode { Token(Token), Call(Tagged), + Nodes(Tagged>), Delimited(Tagged), Pipeline(Tagged), - Operator(Tagged), Flag(Tagged), Member(Tag), Whitespace(Tag), Error(Tagged>), - Path(Tagged), } impl ToDebug for TokenNode { @@ -94,32 +94,33 @@ impl TokenNode { pub fn tag(&self) -> Tag { match self { TokenNode::Token(t) => t.tag(), + TokenNode::Nodes(t) => t.tag(), TokenNode::Call(s) => s.tag(), TokenNode::Delimited(s) => s.tag(), TokenNode::Pipeline(s) => s.tag(), - TokenNode::Operator(s) => s.tag(), TokenNode::Flag(s) => s.tag(), TokenNode::Member(s) => *s, TokenNode::Whitespace(s) => *s, TokenNode::Error(s) => s.tag(), - TokenNode::Path(s) => s.tag(), } } - pub fn type_name(&self) -> String { + pub fn type_name(&self) -> &'static str { match self { TokenNode::Token(t) => t.type_name(), + TokenNode::Nodes(_) => "nodes", TokenNode::Call(_) => "command", TokenNode::Delimited(d) => d.type_name(), TokenNode::Pipeline(_) => "pipeline", - TokenNode::Operator(_) => "operator", TokenNode::Flag(_) => "flag", TokenNode::Member(_) => "member", TokenNode::Whitespace(_) => "whitespace", TokenNode::Error(_) => "error", - TokenNode::Path(_) => "path", } - .to_string() + } + + pub fn tagged_type_name(&self) -> Tagged<&'static str> { + self.type_name().tagged(self.tag()) } pub fn old_debug<'a>(&'a self, source: &'a Text) -> DebugTokenNode<'a> { @@ -134,6 +135,16 @@ impl TokenNode { self.tag().slice(source) } + pub fn get_variable(&self) -> Result<(Tag, Tag), ShellError> { + match self { + TokenNode::Token(Tagged { + item: RawToken::Variable(inner_tag), + tag: outer_tag, + }) => Ok((*outer_tag, *inner_tag)), + _ => Err(ShellError::type_error("variable", self.tagged_type_name())), + } + } + pub fn is_bare(&self) -> bool { match self { TokenNode::Token(Tagged { @@ -144,6 +155,20 @@ impl TokenNode { } } + pub fn as_block(&self) -> Option> { + match self { + TokenNode::Delimited(Tagged { + item: + DelimitedNode { + delimiter, + children, + }, + tag, + }) if *delimiter == Delimiter::Brace => Some((&children[..]).tagged(tag)), + _ => None, + } + } + pub fn is_external(&self) -> bool { match self { TokenNode::Token(Tagged { @@ -181,13 +206,60 @@ impl TokenNode { _ => Err(ShellError::string("unimplemented")), } } + + pub fn is_whitespace(&self) -> bool { + match self { + TokenNode::Whitespace(_) => true, + _ => false, + } + } + + pub fn expect_string(&self) -> (Tag, Tag) { + match self { + TokenNode::Token(Tagged { + item: RawToken::String(inner_tag), + tag: outer_tag, + }) => (*outer_tag, *inner_tag), + other => panic!("Expected string, found {:?}", other), + } + } +} + +#[cfg(test)] +impl TokenNode { + pub fn expect_list(&self) -> Tagged<&[TokenNode]> { + match self { + TokenNode::Nodes(Tagged { item, tag }) => (&item[..]).tagged(tag), + other => panic!("Expected list, found {:?}", other), + } + } + + pub fn expect_var(&self) -> (Tag, Tag) { + match self { + TokenNode::Token(Tagged { + item: RawToken::Variable(inner_tag), + tag: outer_tag, + }) => (*outer_tag, *inner_tag), + other => panic!("Expected var, found {:?}", other), + } + } + + pub fn expect_bare(&self) -> Tag { + match self { + TokenNode::Token(Tagged { + item: RawToken::Bare, + tag, + }) => *tag, + other => panic!("Expected var, found {:?}", other), + } + } } #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] #[get = "pub(crate)"] pub struct DelimitedNode { - delimiter: Delimiter, - children: Vec, + pub(crate) delimiter: Delimiter, + pub(crate) children: Vec, } impl DelimitedNode { @@ -207,6 +279,24 @@ pub enum Delimiter { Square, } +impl Delimiter { + pub(crate) fn open(&self) -> char { + match self { + Delimiter::Paren => '(', + Delimiter::Brace => '{', + Delimiter::Square => '[', + } + } + + pub(crate) fn close(&self) -> char { + match self { + Delimiter::Paren => ')', + Delimiter::Brace => '}', + Delimiter::Square => ']', + } + } +} + #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] #[get = "pub(crate)"] pub struct PathNode { diff --git a/src/parser/parse/token_tree_builder.rs b/src/parser/parse/token_tree_builder.rs index 9a2e6ab721..67298987a4 100644 --- a/src/parser/parse/token_tree_builder.rs +++ b/src/parser/parse/token_tree_builder.rs @@ -3,7 +3,7 @@ use crate::prelude::*; use crate::parser::parse::flag::{Flag, FlagKind}; use crate::parser::parse::operator::Operator; use crate::parser::parse::pipeline::{Pipeline, PipelineElement}; -use crate::parser::parse::token_tree::{DelimitedNode, Delimiter, PathNode, TokenNode}; +use crate::parser::parse::token_tree::{DelimitedNode, Delimiter, TokenNode}; use crate::parser::parse::tokens::{RawNumber, RawToken}; use crate::parser::parse::unit::Unit; use crate::parser::CallNode; @@ -31,60 +31,68 @@ impl TokenTreeBuilder { (node, builder.output) } - pub fn pipeline(input: Vec<(Option<&str>, CurriedCall, Option<&str>)>) -> CurriedToken { - let input: Vec<(Option, CurriedCall, Option)> = input - .into_iter() - .map(|(pre, call, post)| { - ( - pre.map(|s| s.to_string()), - call, - post.map(|s| s.to_string()), - ) - }) - .collect(); + fn build_tagged(&mut self, callback: impl FnOnce(&mut TokenTreeBuilder) -> T) -> Tagged { + let start = self.pos; + let ret = callback(self); + let end = self.pos; + ret.tagged((start, end, self.anchor)) + } + + pub fn pipeline(input: Vec>) -> CurriedToken { Box::new(move |b| { let start = b.pos; - let mut out: Vec = vec![]; + let mut out: Vec> = vec![]; let mut input = input.into_iter().peekable(); - let (pre, call, post) = input + let head = input .next() .expect("A pipeline must contain at least one element"); let pipe = None; - let pre_tag = pre.map(|pre| b.consume_tag(&pre)); - let call = call(b); - let post_tag = post.map(|post| b.consume_tag(&post)); + let head = b.build_tagged(|b| head.into_iter().map(|node| node(b)).collect()); - out.push(PipelineElement::new(pipe, pre_tag, call, post_tag)); + let head_tag: Tag = head.tag; + out.push(PipelineElement::new(pipe, head).tagged(head_tag)); loop { match input.next() { None => break, - Some((pre, call, post)) => { + Some(node) => { + let start = b.pos; let pipe = Some(b.consume_tag("|")); - let pre_span = pre.map(|pre| b.consume_tag(&pre)); - let call = call(b); - let post_span = post.map(|post| b.consume_tag(&post)); + let node = + b.build_tagged(|b| node.into_iter().map(|node| node(b)).collect()); + let end = b.pos; - out.push(PipelineElement::new(pipe, pre_span, call, post_span)); + out.push(PipelineElement::new(pipe, node).tagged((start, end, b.anchor))); } } } let end = b.pos; - TokenTreeBuilder::tagged_pipeline((out, None), (start, end, b.anchor)) + TokenTreeBuilder::tagged_pipeline(out, (start, end, b.anchor)) }) } - pub fn tagged_pipeline( - input: (Vec, Option), - tag: impl Into, - ) -> TokenNode { - TokenNode::Pipeline(Pipeline::new(input.0, input.1.into()).tagged(tag.into())) + pub fn tagged_pipeline(input: Vec>, tag: impl Into) -> TokenNode { + TokenNode::Pipeline(Pipeline::new(input).tagged(tag.into())) + } + + pub fn token_list(input: Vec) -> CurriedToken { + Box::new(move |b| { + let start = b.pos; + let tokens = input.into_iter().map(|i| i(b)).collect(); + let end = b.pos; + + TokenTreeBuilder::tagged_token_list(tokens, (start, end, b.anchor)) + }) + } + + pub fn tagged_token_list(input: Vec, tag: impl Into) -> TokenNode { + TokenNode::Nodes(input.tagged(tag)) } pub fn op(input: impl Into) -> CurriedToken { @@ -100,7 +108,7 @@ impl TokenTreeBuilder { } pub fn tagged_op(input: impl Into, tag: impl Into) -> TokenNode { - TokenNode::Operator(input.into().tagged(tag.into())) + TokenNode::Token(RawToken::Operator(input.into()).tagged(tag.into())) } pub fn string(input: impl Into) -> CurriedToken { @@ -168,8 +176,23 @@ impl TokenTreeBuilder { TokenNode::Token(RawToken::ExternalWord.tagged(input.into())) } - pub fn tagged_external(input: impl Into, tag: impl Into) -> TokenNode { - TokenNode::Token(RawToken::ExternalCommand(input.into()).tagged(tag.into())) + pub fn external_command(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (outer_start, _) = b.consume("^"); + let (inner_start, end) = b.consume(&input); + b.pos = end; + + TokenTreeBuilder::tagged_external_command( + (inner_start, end, b.anchor), + (outer_start, end, b.anchor), + ) + }) + } + + pub fn tagged_external_command(inner: impl Into, outer: impl Into) -> TokenNode { + TokenNode::Token(RawToken::ExternalCommand(inner.into()).tagged(outer.into())) } pub fn int(input: impl Into) -> CurriedToken { @@ -229,29 +252,6 @@ impl TokenTreeBuilder { TokenNode::Token(RawToken::Size(int, unit).tagged(tag.into())) } - pub fn path(head: CurriedToken, tail: Vec) -> CurriedToken { - Box::new(move |b| { - let start = b.pos; - let head = head(b); - - let mut output = vec![]; - - for item in tail { - b.consume("."); - - output.push(item(b)); - } - - let end = b.pos; - - TokenTreeBuilder::tagged_path((head, output), (start, end, b.anchor)) - }) - } - - pub fn tagged_path(input: (TokenNode, Vec), tag: impl Into) -> TokenNode { - TokenNode::Path(PathNode::new(Box::new(input.0), input.1).tagged(tag.into())) - } - pub fn var(input: impl Into) -> CurriedToken { let input = input.into(); diff --git a/src/parser/parse/tokens.rs b/src/parser/parse/tokens.rs index d796a8fcb7..77a856af3f 100644 --- a/src/parser/parse/tokens.rs +++ b/src/parser/parse/tokens.rs @@ -1,4 +1,5 @@ use crate::parser::parse::unit::*; +use crate::parser::Operator; use crate::prelude::*; use crate::{Tagged, Text}; use std::fmt; @@ -7,6 +8,7 @@ use std::str::FromStr; #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum RawToken { Number(RawNumber), + Operator(Operator), Size(RawNumber, Unit), String(Tag), Variable(Tag), @@ -49,12 +51,13 @@ impl RawToken { pub fn type_name(&self) -> &'static str { match self { RawToken::Number(_) => "Number", + RawToken::Operator(..) => "operator", RawToken::Size(..) => "Size", RawToken::String(_) => "String", - RawToken::Variable(_) => "Variable", - RawToken::ExternalCommand(_) => "ExternalCommand", - RawToken::ExternalWord => "ExternalWord", - RawToken::GlobPattern => "GlobPattern", + RawToken::Variable(_) => "variable", + RawToken::ExternalCommand(_) => "external command", + RawToken::ExternalWord => "external word", + RawToken::GlobPattern => "glob pattern", RawToken::Bare => "String", } } diff --git a/src/parser/parse_command.rs b/src/parser/parse_command.rs index 36ba82f8e5..d383689fd9 100644 --- a/src/parser/parse_command.rs +++ b/src/parser/parse_command.rs @@ -1,92 +1,35 @@ -use crate::context::Context; use crate::errors::{ArgumentError, ShellError}; +use crate::parser::hir::syntax_shape::{expand_expr, spaced}; use crate::parser::registry::{NamedType, PositionalType, Signature}; -use crate::parser::{baseline_parse_tokens, CallNode}; +use crate::parser::TokensIterator; use crate::parser::{ - hir::{self, NamedArguments}, - Flag, RawToken, TokenNode, + hir::{self, ExpandContext, NamedArguments}, + Flag, }; use crate::traits::ToDebug; -use crate::{Tag, Tagged, TaggedItem, Text}; +use crate::{Tag, Tagged, Text}; use log::trace; -pub fn parse_command( +pub fn parse_command_tail( config: &Signature, - context: &Context, - call: &Tagged, - source: &Text, -) -> Result { - let Tagged { item: raw_call, .. } = call; - - trace!("Processing {:?}", config); - - let head = parse_command_head(call.head())?; - - let children: Option> = raw_call.children().as_ref().map(|nodes| { - nodes - .iter() - .cloned() - .filter(|node| match node { - TokenNode::Whitespace(_) => false, - _ => true, - }) - .collect() - }); - - match parse_command_tail(&config, context, children, source, call.tag())? { - None => Ok(hir::Call::new(Box::new(head), None, None)), - Some((positional, named)) => Ok(hir::Call::new(Box::new(head), positional, named)), - } -} - -fn parse_command_head(head: &TokenNode) -> Result { - match head { - TokenNode::Token( - spanned @ Tagged { - item: RawToken::Bare, - .. - }, - ) => Ok(spanned.map(|_| hir::RawExpression::Literal(hir::Literal::Bare))), - - TokenNode::Token(Tagged { - item: RawToken::String(inner_tag), - tag, - }) => Ok(hir::RawExpression::Literal(hir::Literal::String(*inner_tag)).tagged(*tag)), - - other => Err(ShellError::unexpected(&format!( - "command head -> {:?}", - other - ))), - } -} - -fn parse_command_tail( - config: &Signature, - context: &Context, - tail: Option>, - source: &Text, + context: &ExpandContext, + tail: &mut TokensIterator, command_tag: Tag, ) -> Result>, Option)>, ShellError> { - let tail = &mut match &tail { - None => hir::TokensIterator::new(&[]), - Some(tail) => hir::TokensIterator::new(tail), - }; - let mut named = NamedArguments::new(); - - trace_remaining("nodes", tail.clone(), source); + trace_remaining("nodes", tail.clone(), context.source()); for (name, kind) in &config.named { trace!(target: "nu::parse", "looking for {} : {:?}", name, kind); match kind { NamedType::Switch => { - let flag = extract_switch(name, tail, source); + let flag = extract_switch(name, tail, context.source()); named.insert_switch(name, flag); } NamedType::Mandatory(syntax_type) => { - match extract_mandatory(config, name, tail, source, command_tag) { + match extract_mandatory(config, name, tail, context.source(), command_tag) { Err(err) => return Err(err), // produce a correct diagnostic Ok((pos, flag)) => { tail.move_to(pos); @@ -99,42 +42,47 @@ fn parse_command_tail( )); } - let expr = - hir::baseline_parse_next_expr(tail, context, source, *syntax_type)?; + let expr = expand_expr(&spaced(*syntax_type), tail, context)?; tail.restart(); named.insert_mandatory(name, expr); } } } - NamedType::Optional(syntax_type) => match extract_optional(name, tail, source) { - Err(err) => return Err(err), // produce a correct diagnostic - Ok(Some((pos, flag))) => { - tail.move_to(pos); + NamedType::Optional(syntax_type) => { + match extract_optional(name, tail, context.source()) { + Err(err) => return Err(err), // produce a correct diagnostic + Ok(Some((pos, flag))) => { + tail.move_to(pos); - if tail.at_end() { - return Err(ShellError::argument_error( - config.name.clone(), - ArgumentError::MissingValueForName(name.to_string()), - flag.tag(), - )); + if tail.at_end() { + return Err(ShellError::argument_error( + config.name.clone(), + ArgumentError::MissingValueForName(name.to_string()), + flag.tag(), + )); + } + + let expr = expand_expr(&spaced(*syntax_type), tail, context); + + match expr { + Err(_) => named.insert_optional(name, None), + Ok(expr) => named.insert_optional(name, Some(expr)), + } + + tail.restart(); } - let expr = hir::baseline_parse_next_expr(tail, context, source, *syntax_type)?; - - tail.restart(); - named.insert_optional(name, Some(expr)); + Ok(None) => { + tail.restart(); + named.insert_optional(name, None); + } } - - Ok(None) => { - tail.restart(); - named.insert_optional(name, None); - } - }, + } }; } - trace_remaining("after named", tail.clone(), source); + trace_remaining("after named", tail.clone(), context.source()); let mut positional = vec![]; @@ -143,7 +91,7 @@ fn parse_command_tail( match arg { PositionalType::Mandatory(..) => { - if tail.len() == 0 { + if tail.at_end() { return Err(ShellError::argument_error( config.name.clone(), ArgumentError::MissingMandatoryPositional(arg.name().to_string()), @@ -153,25 +101,36 @@ fn parse_command_tail( } PositionalType::Optional(..) => { - if tail.len() == 0 { + if tail.at_end() { break; } } } - let result = hir::baseline_parse_next_expr(tail, context, source, arg.syntax_type())?; + let result = expand_expr(&spaced(arg.syntax_type()), tail, context)?; positional.push(result); } - trace_remaining("after positional", tail.clone(), source); + trace_remaining("after positional", tail.clone(), context.source()); if let Some(syntax_type) = config.rest_positional { - let remainder = baseline_parse_tokens(tail, context, source, syntax_type)?; - positional.extend(remainder); + let mut out = vec![]; + + loop { + if tail.at_end_possible_ws() { + break; + } + + let next = expand_expr(&spaced(syntax_type), tail, context)?; + + out.push(next); + } + + positional.extend(out); } - trace_remaining("after rest", tail.clone(), source); + trace_remaining("after rest", tail.clone(), context.source()); trace!("Constructed positional={:?} named={:?}", positional, named); diff --git a/src/parser/registry.rs b/src/parser/registry.rs index 955a1a04c9..888e5ae1e9 100644 --- a/src/parser/registry.rs +++ b/src/parser/registry.rs @@ -1,11 +1,11 @@ // TODO: Temporary redirect pub(crate) use crate::context::CommandRegistry; use crate::evaluate::{evaluate_baseline_expr, Scope}; -use crate::parser::{hir, hir::SyntaxShape, parse_command, CallNode}; +use crate::parser::{hir, hir::SyntaxShape}; use crate::prelude::*; use derive_new::new; use indexmap::IndexMap; -use log::trace; + use serde::{Deserialize, Serialize}; use std::fmt; @@ -271,21 +271,6 @@ impl<'a> Iterator for PositionalIter<'a> { } } -impl Signature { - pub(crate) fn parse_args( - &self, - call: &Tagged, - context: &Context, - source: &Text, - ) -> Result { - let args = parse_command(self, context, call, source)?; - - trace!("parsed args: {:?}", args); - - Ok(args) - } -} - pub(crate) fn evaluate_args( call: &hir::Call, registry: &CommandRegistry, diff --git a/src/plugins/add.rs b/src/plugins/add.rs index 03e1d42828..997400d67f 100644 --- a/src/plugins/add.rs +++ b/src/plugins/add.rs @@ -1,10 +1,13 @@ +use itertools::Itertools; use nu::{ - serve_plugin, CallInfo, Plugin, Primitive, ReturnSuccess, ReturnValue, ShellError, Signature, - SyntaxShape, Tagged, Value, + serve_plugin, CallInfo, Plugin, ReturnSuccess, ReturnValue, ShellError, Signature, SyntaxShape, + Tagged, Value, }; +pub type ColumnPath = Vec>; + struct Add { - field: Option, + field: Option, value: Option, } impl Add { @@ -19,12 +22,13 @@ impl Add { let value_tag = value.tag(); match (value.item, self.value.clone()) { (obj @ Value::Row(_), Some(v)) => match &self.field { - Some(f) => match obj.insert_data_at_path(value_tag, &f, v) { + Some(f) => match obj.insert_data_at_column_path(value_tag, &f, v) { Some(v) => return Ok(v), None => { return Err(ShellError::string(format!( "add could not find place to insert field {:?} {}", - obj, f + obj, + f.iter().map(|i| &i.item).join(".") ))) } }, @@ -44,7 +48,7 @@ impl Plugin for Add { fn config(&mut self) -> Result { Ok(Signature::build("add") .desc("Add a new field to the table.") - .required("Field", SyntaxShape::String) + .required("Field", SyntaxShape::ColumnPath) .required("Value", SyntaxShape::String) .rest(SyntaxShape::String) .filter()) @@ -53,12 +57,13 @@ impl Plugin for Add { fn begin_filter(&mut self, call_info: CallInfo) -> Result, ShellError> { if let Some(args) = call_info.args.positional { match &args[0] { - Tagged { - item: Value::Primitive(Primitive::String(s)), + table @ Tagged { + item: Value::Table(_), .. } => { - self.field = Some(s.clone()); + self.field = Some(table.as_column_path()?.item); } + _ => { return Err(ShellError::string(format!( "Unrecognized type in params: {:?}", diff --git a/src/plugins/edit.rs b/src/plugins/edit.rs index db116fedf5..6d35530ef5 100644 --- a/src/plugins/edit.rs +++ b/src/plugins/edit.rs @@ -1,10 +1,12 @@ use nu::{ - serve_plugin, CallInfo, Plugin, Primitive, ReturnSuccess, ReturnValue, ShellError, Signature, - SyntaxShape, Tagged, Value, + serve_plugin, CallInfo, Plugin, ReturnSuccess, ReturnValue, ShellError, Signature, SyntaxShape, + Tagged, Value, }; +pub type ColumnPath = Vec>; + struct Edit { - field: Option, + field: Option, value: Option, } impl Edit { @@ -19,7 +21,7 @@ impl Edit { let value_tag = value.tag(); match (value.item, self.value.clone()) { (obj @ Value::Row(_), Some(v)) => match &self.field { - Some(f) => match obj.replace_data_at_path(value_tag, &f, v) { + Some(f) => match obj.replace_data_at_column_path(value_tag, &f, v) { Some(v) => return Ok(v), None => { return Err(ShellError::string( @@ -43,7 +45,7 @@ impl Plugin for Edit { fn config(&mut self) -> Result { Ok(Signature::build("edit") .desc("Edit an existing column to have a new value.") - .required("Field", SyntaxShape::String) + .required("Field", SyntaxShape::ColumnPath) .required("Value", SyntaxShape::String) .filter()) } @@ -51,11 +53,11 @@ impl Plugin for Edit { fn begin_filter(&mut self, call_info: CallInfo) -> Result, ShellError> { if let Some(args) = call_info.args.positional { match &args[0] { - Tagged { - item: Value::Primitive(Primitive::String(s)), + table @ Tagged { + item: Value::Table(_), .. } => { - self.field = Some(s.clone()); + self.field = Some(table.as_column_path()?.item); } _ => { return Err(ShellError::string(format!( diff --git a/src/plugins/inc.rs b/src/plugins/inc.rs index ecab03dc97..4e6f6f0f64 100644 --- a/src/plugins/inc.rs +++ b/src/plugins/inc.rs @@ -14,8 +14,10 @@ pub enum SemVerAction { Patch, } +pub type ColumnPath = Vec>; + struct Inc { - field: Option, + field: Option, error: Option, action: Option, } @@ -85,16 +87,17 @@ impl Inc { } Value::Row(_) => match self.field { Some(ref f) => { - let replacement = match value.item.get_data_by_path(value.tag(), f) { + let replacement = match value.item.get_data_by_column_path(value.tag(), f) { Some(result) => self.inc(result.map(|x| x.clone()))?, None => { return Err(ShellError::string("inc could not find field to replace")) } }; - match value - .item - .replace_data_at_path(value.tag(), f, replacement.item.clone()) - { + match value.item.replace_data_at_column_path( + value.tag(), + f, + replacement.item.clone(), + ) { Some(v) => return Ok(v), None => { return Err(ShellError::string("inc could not find field to replace")) @@ -120,7 +123,7 @@ impl Plugin for Inc { .switch("major") .switch("minor") .switch("patch") - .rest(SyntaxShape::String) + .rest(SyntaxShape::ColumnPath) .filter()) } @@ -138,11 +141,11 @@ impl Plugin for Inc { if let Some(args) = call_info.args.positional { for arg in args { match arg { - Tagged { - item: Value::Primitive(Primitive::String(s)), + table @ Tagged { + item: Value::Table(_), .. } => { - self.field = Some(s); + self.field = Some(table.as_column_path()?.item); } _ => { return Err(ShellError::string(format!( @@ -209,8 +212,13 @@ mod tests { } fn with_parameter(&mut self, name: &str) -> &mut Self { + let fields: Vec> = name + .split(".") + .map(|s| Value::string(s.to_string()).tagged(Tag::unknown_span(self.anchor))) + .collect(); + self.positionals - .push(Value::string(name.to_string()).tagged(Tag::unknown_span(self.anchor))); + .push(Value::Table(fields).tagged(Tag::unknown_span(self.anchor))); self } @@ -297,7 +305,12 @@ mod tests { ) .is_ok()); - assert_eq!(plugin.field, Some("package.version".to_string())); + assert_eq!( + plugin + .field + .map(|f| f.into_iter().map(|f| f.item).collect()), + Some(vec!["package".to_string(), "version".to_string()]) + ); } #[test] diff --git a/src/plugins/str.rs b/src/plugins/str.rs index 4b74914f09..7bd35733da 100644 --- a/src/plugins/str.rs +++ b/src/plugins/str.rs @@ -1,6 +1,6 @@ use nu::{ serve_plugin, CallInfo, Plugin, Primitive, ReturnSuccess, ReturnValue, ShellError, Signature, - SyntaxShape, Tagged, Value, + SyntaxShape, Tagged, TaggedItem, Value, }; #[derive(Debug, Eq, PartialEq)] @@ -10,8 +10,10 @@ enum Action { ToInteger, } +pub type ColumnPath = Vec>; + struct Str { - field: Option, + field: Option, params: Option>, error: Option, action: Option, @@ -43,8 +45,8 @@ impl Str { Ok(applied) } - fn for_field(&mut self, field: &str) { - self.field = Some(String::from(field)); + fn for_field(&mut self, column_path: ColumnPath) { + self.field = Some(column_path); } fn permit(&mut self) -> bool { @@ -92,14 +94,15 @@ impl Str { } Value::Row(_) => match self.field { Some(ref f) => { - let replacement = match value.item.get_data_by_path(value.tag(), f) { + let replacement = match value.item.get_data_by_column_path(value.tag(), f) { Some(result) => self.strutils(result.map(|x| x.clone()))?, None => return Ok(Tagged::from_item(Value::nothing(), value.tag)), }; - match value - .item - .replace_data_at_path(value.tag(), f, replacement.item.clone()) - { + match value.item.replace_data_at_column_path( + value.tag(), + f, + replacement.item.clone(), + ) { Some(v) => return Ok(v), None => { return Err(ShellError::string("str could not find field to replace")) @@ -127,7 +130,7 @@ impl Plugin for Str { .switch("downcase") .switch("upcase") .switch("to-int") - .rest(SyntaxShape::Member) + .rest(SyntaxShape::ColumnPath) .filter()) } @@ -148,15 +151,21 @@ impl Plugin for Str { match possible_field { Tagged { item: Value::Primitive(Primitive::String(s)), - .. + tag, } => match self.action { Some(Action::Downcase) | Some(Action::Upcase) | Some(Action::ToInteger) | None => { - self.for_field(&s); + self.for_field(vec![s.clone().tagged(tag)]); } }, + table @ Tagged { + item: Value::Table(_), + .. + } => { + self.field = Some(table.as_column_path()?.item); + } _ => { return Err(ShellError::string(format!( "Unrecognized type in params: {:?}", @@ -227,8 +236,13 @@ mod tests { } fn with_parameter(&mut self, name: &str) -> &mut Self { + let fields: Vec> = name + .split(".") + .map(|s| Value::string(s.to_string()).tagged(Tag::unknown_span(self.anchor))) + .collect(); + self.positionals - .push(Value::string(name.to_string()).tagged(Tag::unknown())); + .push(Value::Table(fields).tagged(Tag::unknown_span(self.anchor))); self } @@ -303,7 +317,12 @@ mod tests { ) .is_ok()); - assert_eq!(plugin.field, Some("package.description".to_string())); + assert_eq!( + plugin + .field + .map(|f| f.into_iter().map(|f| f.item).collect()), + Some(vec!["package".to_string(), "description".to_string()]) + ) } #[test] diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 6fb4544352..85591cf047 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -1,3 +1,4 @@ +use crate::parser::hir::TokensIterator; use crate::parser::nom_input; use crate::parser::parse::token_tree::TokenNode; use crate::parser::parse::tokens::RawToken; @@ -77,16 +78,12 @@ impl Highlighter for Helper { Ok(v) => v, }; - let Pipeline { parts, post_ws } = pipeline; + let Pipeline { parts } = pipeline; let mut iter = parts.into_iter(); loop { match iter.next() { None => { - if let Some(ws) = post_ws { - out.push_str(ws.slice(line)); - } - return Cow::Owned(out); } Some(token) => { @@ -107,13 +104,12 @@ impl Highlighter for Helper { fn paint_token_node(token_node: &TokenNode, line: &str) -> String { let styled = match token_node { TokenNode::Call(..) => Color::Cyan.bold().paint(token_node.tag().slice(line)), + TokenNode::Nodes(..) => Color::Green.bold().paint(token_node.tag().slice(line)), TokenNode::Whitespace(..) => Color::White.normal().paint(token_node.tag().slice(line)), TokenNode::Flag(..) => Color::Black.bold().paint(token_node.tag().slice(line)), TokenNode::Member(..) => Color::Yellow.bold().paint(token_node.tag().slice(line)), - TokenNode::Path(..) => Color::Green.bold().paint(token_node.tag().slice(line)), TokenNode::Error(..) => Color::Red.bold().paint(token_node.tag().slice(line)), TokenNode::Delimited(..) => Color::White.paint(token_node.tag().slice(line)), - TokenNode::Operator(..) => Color::White.normal().paint(token_node.tag().slice(line)), TokenNode::Pipeline(..) => Color::Blue.normal().paint(token_node.tag().slice(line)), TokenNode::Token(Tagged { item: RawToken::Number(..), @@ -147,6 +143,10 @@ fn paint_token_node(token_node: &TokenNode, line: &str) -> String { item: RawToken::ExternalWord, .. }) => Color::Black.bold().paint(token_node.tag().slice(line)), + TokenNode::Token(Tagged { + item: RawToken::Operator(..), + .. + }) => Color::Black.bold().paint(token_node.tag().slice(line)), }; styled.to_string() @@ -159,25 +159,19 @@ fn paint_pipeline_element(pipeline_element: &PipelineElement, line: &str) -> Str styled.push_str(&Color::Purple.paint("|")); } - if let Some(ws) = pipeline_element.pre_ws { - styled.push_str(&Color::White.normal().paint(ws.slice(line))); - } + let mut tokens = + TokensIterator::new(&pipeline_element.tokens, pipeline_element.tokens.tag, false); + let head = tokens.next(); - styled.push_str( - &Color::Cyan - .bold() - .paint(pipeline_element.call().head().tag().slice(line)) - .to_string(), - ); - - if let Some(children) = pipeline_element.call().children() { - for child in children { - styled.push_str(&paint_token_node(child, line)); + match head { + None => return styled, + Some(head) => { + styled.push_str(&Color::Cyan.bold().paint(head.tag().slice(line)).to_string()) } } - if let Some(ws) = pipeline_element.post_ws { - styled.push_str(&Color::White.normal().paint(ws.slice(line))); + for token in tokens { + styled.push_str(&paint_token_node(token, line)); } styled.to_string() diff --git a/tests/command_open_tests.rs b/tests/command_open_tests.rs index 54dc7ad54d..e9047883cf 100644 --- a/tests/command_open_tests.rs +++ b/tests/command_open_tests.rs @@ -212,7 +212,7 @@ fn open_can_parse_ini() { fn open_can_parse_utf16_ini() { let actual = nu!( cwd: "tests/fixtures/formats", - "open utf16.ini | get .ShellClassInfo | get IconIndex | echo $it" + "open utf16.ini | get '.ShellClassInfo' | get IconIndex | echo $it" ); assert_eq!(actual, "-236") diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs index 04fd889925..199038b531 100644 --- a/tests/helpers/mod.rs +++ b/tests/helpers/mod.rs @@ -93,6 +93,7 @@ macro_rules! nu { .write_all(commands.as_bytes()) .expect("couldn't write to stdin"); + let output = process .wait_with_output() .expect("couldn't read from stdout"); From c2c10e2bc0254497c25968728641c4f1b9b7f135 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Sun, 6 Oct 2019 13:22:50 -0700 Subject: [PATCH 2/2] Overhaul the coloring system This commit replaces the previous naive coloring system with a coloring system that is more aligned with the parser. The main benefit of this change is that it allows us to use parsing rules to decide how to color tokens. For example, consider the following syntax: ``` $ ps | where cpu > 10 ``` Ideally, we could color `cpu` like a column name and not a string, because `cpu > 10` is a shorthand block syntax that expands to `{ $it.cpu > 10 }`. The way that we know that it's a shorthand block is that the `where` command declares that its first parameter is a `SyntaxShape::Block`, which allows the shorthand block form. In order to accomplish this, we need to color the tokens in a way that corresponds to their expanded semantics, which means that high-fidelity coloring requires expansion. This commit adds a `ColorSyntax` trait that corresponds to the `ExpandExpression` trait. The semantics are fairly similar, with a few differences. First `ExpandExpression` consumes N tokens and returns a single `hir::Expression`. `ColorSyntax` consumes N tokens and writes M `FlatShape` tokens to the output. Concretely, for syntax like `[1 2 3]` - `ExpandExpression` takes a single token node and produces a single `hir::Expression` - `ColorSyntax` takes the same token node and emits 7 `FlatShape`s (open delimiter, int, whitespace, int, whitespace, int, close delimiter) Second, `ColorSyntax` is more willing to plow through failures than `ExpandExpression`. In particular, consider syntax like ``` $ ps | where cpu > ``` In this case - `ExpandExpression` will see that the `where` command is expecting a block, see that it's not a literal block and try to parse it as a shorthand block. It will successfully find a member followed by an infix operator, but not a following expression. That means that the entire pipeline part fails to parse and is a syntax error. - `ColorSyntax` will also try to parse it as a shorthand block and ultimately fail, but it will fall back to "backoff coloring mode", which parsing any unidentified tokens in an unfallible, simple way. In this case, `cpu` will color as a string and `>` will color as an operator. Finally, it's very important that coloring a pipeline infallibly colors the entire string, doesn't fail, and doesn't get stuck in an infinite loop. In order to accomplish this, this PR separates `ColorSyntax`, which is infallible from `FallibleColorSyntax`, which might fail. This allows the type system to let us know if our coloring rules bottom out at at an infallible rule. It's not perfect: it's still possible for the coloring process to get stuck or consume tokens non-atomically. I intend to reduce the opportunity for those problems in a future commit. In the meantime, the current system catches a number of mistakes (like trying to use a fallible coloring rule in a loop without thinking about the possibility that it will never terminate). --- Cargo.toml | 2 +- src/cli.rs | 95 +-- src/commands/autoview.rs | 32 +- src/commands/classified.rs | 29 +- src/commands/config.rs | 13 +- src/commands/fetch.rs | 12 +- src/commands/open.rs | 12 +- src/commands/plugin.rs | 12 +- src/commands/post.rs | 16 +- src/commands/save.rs | 5 +- src/commands/to_csv.rs | 57 +- src/commands/to_tsv.rs | 70 +- src/context.rs | 10 +- src/data/base.rs | 48 +- src/data/config.rs | 22 +- src/data/meta.rs | 29 + src/errors.rs | 108 +-- src/parser.rs | 6 +- src/parser/hir/expand_external_tokens.rs | 88 ++- src/parser/hir/syntax_shape.rs | 655 +++++++++++++++++- src/parser/hir/syntax_shape/block.rs | 172 ++++- src/parser/hir/syntax_shape/expression.rs | 255 +++++-- .../hir/syntax_shape/expression/atom.rs | 541 +++++++++++++++ .../hir/syntax_shape/expression/delimited.rs | 67 +- .../hir/syntax_shape/expression/file_path.rs | 94 +-- .../hir/syntax_shape/expression/list.rs | 141 +++- .../hir/syntax_shape/expression/number.rs | 108 +-- .../hir/syntax_shape/expression/pattern.rs | 34 +- .../hir/syntax_shape/expression/string.rs | 36 +- .../hir/syntax_shape/expression/unit.rs | 31 +- .../syntax_shape/expression/variable_path.rs | 346 ++++++++- src/parser/hir/syntax_shape/flat_shape.rs | 95 +++ src/parser/hir/tokens_iterator.rs | 142 +++- src/parser/parse/flag.rs | 16 +- src/parser/parse/parser.rs | 53 +- src/parser/parse/token_tree.rs | 55 +- src/parser/parse/token_tree_builder.rs | 120 ++-- src/parser/parse/tokens.rs | 103 ++- src/parser/parse_command.rs | 232 ++++++- src/plugin.rs | 8 +- src/plugins/add.rs | 35 +- src/plugins/edit.rs | 26 +- src/plugins/embed.rs | 11 +- src/plugins/inc.rs | 41 +- src/plugins/match.rs | 44 +- src/plugins/str.rs | 31 +- src/plugins/sum.rs | 25 +- src/prelude.rs | 10 + src/shell/filesystem_shell.rs | 2 +- src/shell/helper.rs | 177 +++-- 50 files changed, 3527 insertions(+), 845 deletions(-) create mode 100644 src/parser/hir/syntax_shape/expression/atom.rs create mode 100644 src/parser/hir/syntax_shape/flat_shape.rs diff --git a/Cargo.toml b/Cargo.toml index 66bd695c08..80a077dd88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ textview = ["syntect", "onig_sys", "crossterm"] binaryview = ["image", "crossterm"] sys = ["heim", "battery"] ps = ["heim"] -trace = ["nom-tracable/trace"] +# trace = ["nom-tracable/trace"] all = ["raw-key", "textview", "binaryview", "sys", "ps", "clipboard", "ptree"] [dependencies.rusqlite] diff --git a/src/cli.rs b/src/cli.rs index 6a35608d91..6c1ba5ef93 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -14,9 +14,9 @@ use crate::git::current_branch; use crate::parser::registry::Signature; use crate::parser::{ hir, - hir::syntax_shape::{CommandHeadShape, CommandSignature, ExpandSyntax}, + hir::syntax_shape::{expand_syntax, PipelineShape}, hir::{expand_external_tokens::expand_external_tokens, tokens_iterator::TokensIterator}, - parse_command_tail, Pipeline, PipelineElement, TokenNode, + TokenNode, }; use crate::prelude::*; @@ -99,11 +99,17 @@ fn load_plugin(path: &std::path::Path, context: &mut Context) -> Result<(), Shel }, Err(e) => { trace!("incompatible plugin {:?}", input); - Err(ShellError::string(format!("Error: {:?}", e))) + Err(ShellError::untagged_runtime_error(format!( + "Error: {:?}", + e + ))) } } } - Err(e) => Err(ShellError::string(format!("Error: {:?}", e))), + Err(e) => Err(ShellError::untagged_runtime_error(format!( + "Error: {:?}", + e + ))), }; let _ = child.wait(); @@ -319,6 +325,7 @@ pub async fn cli() -> Result<(), Box> { )]); } } + let _ = load_plugins(&mut context); let config = Config::builder().color_mode(ColorMode::Forced).build(); @@ -347,9 +354,7 @@ pub async fn cli() -> Result<(), Box> { let cwd = context.shell_manager.path(); - rl.set_helper(Some(crate::shell::Helper::new( - context.shell_manager.clone(), - ))); + rl.set_helper(Some(crate::shell::Helper::new(context.clone()))); let edit_mode = config::config(Tag::unknown())? .get("edit_mode") @@ -476,7 +481,7 @@ async fn process_line(readline: Result, ctx: &mut Context Ok(line) => { let line = chomp_newline(line); - let result = match crate::parser::parse(&line, uuid::Uuid::new_v4()) { + let result = match crate::parser::parse(&line, uuid::Uuid::nil()) { Err(err) => { return LineResult::Error(line.to_string(), err); } @@ -614,74 +619,14 @@ fn classify_pipeline( context: &Context, source: &Text, ) -> Result { - let pipeline = pipeline.as_pipeline()?; + let mut pipeline_list = vec![pipeline.clone()]; + let mut iterator = TokensIterator::all(&mut pipeline_list, pipeline.tag()); - let Pipeline { parts, .. } = pipeline; - - let commands: Result, ShellError> = parts - .iter() - .map(|item| classify_command(&item, context, &source)) - .collect(); - - Ok(ClassifiedPipeline { - commands: commands?, - }) -} - -fn classify_command( - command: &Tagged, - context: &Context, - source: &Text, -) -> Result { - let mut iterator = TokensIterator::new(&command.tokens.item, command.tag, true); - - let head = CommandHeadShape - .expand_syntax(&mut iterator, &context.expand_context(source, command.tag))?; - - match &head { - CommandSignature::Expression(_) => Err(ShellError::syntax_error( - "Unexpected expression in command position".tagged(command.tag), - )), - - // If the command starts with `^`, treat it as an external command no matter what - CommandSignature::External(name) => { - let name_str = name.slice(source); - - external_command(&mut iterator, source, name_str.tagged(name)) - } - - CommandSignature::LiteralExternal { outer, inner } => { - let name_str = inner.slice(source); - - external_command(&mut iterator, source, name_str.tagged(outer)) - } - - CommandSignature::Internal(command) => { - let tail = parse_command_tail( - &command.signature(), - &context.expand_context(source, command.tag), - &mut iterator, - command.tag, - )?; - - let (positional, named) = match tail { - None => (None, None), - Some((positional, named)) => (positional, named), - }; - - let call = hir::Call { - head: Box::new(head.to_expression()), - positional, - named, - }; - - Ok(ClassifiedCommand::Internal(InternalCommand::new( - command.name().to_string(), - command.tag, - call, - ))) - } - } + expand_syntax( + &PipelineShape, + &mut iterator, + &context.expand_context(source, pipeline.tag()), + ) } // Classify this command as an external command, which doesn't give special meaning diff --git a/src/commands/autoview.rs b/src/commands/autoview.rs index 57ab6269b3..29e7d18121 100644 --- a/src/commands/autoview.rs +++ b/src/commands/autoview.rs @@ -58,21 +58,21 @@ pub fn autoview( } } }; - // } else if is_single_origined_text_value(&input) { - // let text = context.get_command("textview"); - // if let Some(text) = text { - // let result = text.run(raw.with_input(input), &context.commands); - // result.collect::>().await; - // } else { - // for i in input { - // match i.item { - // Value::Primitive(Primitive::String(s)) => { - // println!("{}", s); - // } - // _ => {} - // } - // } - // } + } else if is_single_anchored_text_value(&input) { + let text = context.get_command("textview"); + if let Some(text) = text { + let result = text.run(raw.with_input(input), &context.commands, false); + result.collect::>().await; + } else { + for i in input { + match i.item { + Value::Primitive(Primitive::String(s)) => { + println!("{}", s); + } + _ => {} + } + } + } } else if is_single_text_value(&input) { for i in input { match i.item { @@ -112,7 +112,7 @@ fn is_single_text_value(input: &Vec>) -> bool { } #[allow(unused)] -fn is_single_origined_text_value(input: &Vec>) -> bool { +fn is_single_anchored_text_value(input: &Vec>) -> bool { if input.len() != 1 { return false; } diff --git a/src/commands/classified.rs b/src/commands/classified.rs index d30025b944..c73a56fee4 100644 --- a/src/commands/classified.rs +++ b/src/commands/classified.rs @@ -72,6 +72,7 @@ impl ClassifiedInputStream { } } +#[derive(Debug)] pub(crate) struct ClassifiedPipeline { pub(crate) commands: Vec, } @@ -117,15 +118,19 @@ impl InternalCommand { let command = context.expect_command(&self.name); - let result = context.run_command( - command, - self.name_tag.clone(), - context.source_map.clone(), - self.args, - &source, - objects, - is_first_command, - ); + let result = { + let source_map = context.source_map.lock().unwrap().clone(); + + context.run_command( + command, + self.name_tag.clone(), + source_map, + self.args, + &source, + objects, + is_first_command, + ) + }; let result = trace_out_stream!(target: "nu::trace_stream::internal", source: &source, "output" = result); let mut result = result.values; @@ -253,7 +258,11 @@ impl ExternalCommand { tag, )); } else { - return Err(ShellError::string("Error: $it needs string data")); + return Err(ShellError::labeled_error( + "Error: $it needs string data", + "given something else", + name_tag, + )); } } if !first { diff --git a/src/commands/config.rs b/src/commands/config.rs index 3b36c88fad..337e3437f9 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -70,9 +70,9 @@ pub fn config( if let Some(v) = get { let key = v.to_string(); - let value = result - .get(&key) - .ok_or_else(|| ShellError::string(&format!("Missing key {} in config", key)))?; + let value = result.get(&key).ok_or_else(|| { + ShellError::labeled_error(&format!("Missing key in config"), "key", v.tag()) + })?; let mut results = VecDeque::new(); @@ -120,10 +120,11 @@ pub fn config( result.swap_remove(&key); config::write(&result, &configuration)?; } else { - return Err(ShellError::string(&format!( + return Err(ShellError::labeled_error( "{} does not exist in config", - key - ))); + "key", + v.tag(), + )); } let obj = VecDeque::from_iter(vec![Value::Row(result.into()).tagged(v.tag())]); diff --git a/src/commands/fetch.rs b/src/commands/fetch.rs index 21ef7fbfd9..e7966a61bf 100644 --- a/src/commands/fetch.rs +++ b/src/commands/fetch.rs @@ -44,11 +44,13 @@ fn run( registry: &CommandRegistry, raw_args: &RawCommandArgs, ) -> Result { - let path = match call_info - .args - .nth(0) - .ok_or_else(|| ShellError::string(&format!("No file or directory specified")))? - { + let path = match call_info.args.nth(0).ok_or_else(|| { + ShellError::labeled_error( + "No file or directory specified", + "for command", + call_info.name_tag, + ) + })? { file => file, }; let path_buf = path.as_path()?; diff --git a/src/commands/open.rs b/src/commands/open.rs index 97b0df2744..6ea752e9da 100644 --- a/src/commands/open.rs +++ b/src/commands/open.rs @@ -45,11 +45,13 @@ fn run( let cwd = PathBuf::from(shell_manager.path()); let full_path = PathBuf::from(cwd); - let path = match call_info - .args - .nth(0) - .ok_or_else(|| ShellError::string(&format!("No file or directory specified")))? - { + let path = match call_info.args.nth(0).ok_or_else(|| { + ShellError::labeled_error( + "No file or directory specified", + "for command", + call_info.name_tag, + ) + })? { file => file, }; let path_buf = path.as_path()?; diff --git a/src/commands/plugin.rs b/src/commands/plugin.rs index e769a7b5c7..5dfbe6be5b 100644 --- a/src/commands/plugin.rs +++ b/src/commands/plugin.rs @@ -128,7 +128,7 @@ pub fn filter_plugin( }, Err(e) => { let mut result = VecDeque::new(); - result.push_back(Err(ShellError::string(format!( + result.push_back(Err(ShellError::untagged_runtime_error(format!( "Error while processing begin_filter response: {:?} {}", e, input )))); @@ -138,7 +138,7 @@ pub fn filter_plugin( } Err(e) => { let mut result = VecDeque::new(); - result.push_back(Err(ShellError::string(format!( + result.push_back(Err(ShellError::untagged_runtime_error(format!( "Error while reading begin_filter response: {:?}", e )))); @@ -189,7 +189,7 @@ pub fn filter_plugin( }, Err(e) => { let mut result = VecDeque::new(); - result.push_back(Err(ShellError::string(format!( + result.push_back(Err(ShellError::untagged_runtime_error(format!( "Error while processing end_filter response: {:?} {}", e, input )))); @@ -199,7 +199,7 @@ pub fn filter_plugin( } Err(e) => { let mut result = VecDeque::new(); - result.push_back(Err(ShellError::string(format!( + result.push_back(Err(ShellError::untagged_runtime_error(format!( "Error while reading end_filter: {:?}", e )))); @@ -236,7 +236,7 @@ pub fn filter_plugin( }, Err(e) => { let mut result = VecDeque::new(); - result.push_back(Err(ShellError::string(format!( + result.push_back(Err(ShellError::untagged_runtime_error(format!( "Error while processing filter response: {:?} {}", e, input )))); @@ -246,7 +246,7 @@ pub fn filter_plugin( } Err(e) => { let mut result = VecDeque::new(); - result.push_back(Err(ShellError::string(format!( + result.push_back(Err(ShellError::untagged_runtime_error(format!( "Error while reading filter response: {:?}", e )))); diff --git a/src/commands/post.rs b/src/commands/post.rs index 5a77afd14b..a82f1b42b1 100644 --- a/src/commands/post.rs +++ b/src/commands/post.rs @@ -55,18 +55,14 @@ fn run( raw_args: &RawCommandArgs, ) -> Result { let call_info = call_info.clone(); - let path = match call_info - .args - .nth(0) - .ok_or_else(|| ShellError::string(&format!("No url specified")))? - { + let path = match call_info.args.nth(0).ok_or_else(|| { + ShellError::labeled_error("No url specified", "for command", call_info.name_tag) + })? { file => file.clone(), }; - let body = match call_info - .args - .nth(1) - .ok_or_else(|| ShellError::string(&format!("No body specified")))? - { + let body = match call_info.args.nth(1).ok_or_else(|| { + ShellError::labeled_error("No body specified", "for command", call_info.name_tag) + })? { file => file.clone(), }; let path_str = path.as_string()?; diff --git a/src/commands/save.rs b/src/commands/save.rs index 44e07da5ed..0156fc3557 100644 --- a/src/commands/save.rs +++ b/src/commands/save.rs @@ -150,7 +150,6 @@ fn save( } }, None => { - eprintln!("{:?} {:?}", anchor, source_map); yield Err(ShellError::labeled_error( "Save requires a filepath (2)", "needs path", @@ -213,9 +212,9 @@ fn save( match content { Ok(save_data) => match std::fs::write(full_path, save_data) { Ok(o) => o, - Err(e) => yield Err(ShellError::string(e.to_string())), + Err(e) => yield Err(ShellError::labeled_error(e.to_string(), "for command", name)), }, - Err(e) => yield Err(ShellError::string(e.to_string())), + Err(e) => yield Err(ShellError::labeled_error(e.to_string(), "for command", name)), } }; diff --git a/src/commands/to_csv.rs b/src/commands/to_csv.rs index 1897fb86b7..66121df53e 100644 --- a/src/commands/to_csv.rs +++ b/src/commands/to_csv.rs @@ -32,8 +32,8 @@ impl WholeStreamCommand for ToCSV { } } -pub fn value_to_csv_value(v: &Value) -> Value { - match v { +pub fn value_to_csv_value(v: &Tagged) -> Tagged { + match &v.item { Value::Primitive(Primitive::String(s)) => Value::Primitive(Primitive::String(s.clone())), Value::Primitive(Primitive::Nothing) => Value::Primitive(Primitive::Nothing), Value::Primitive(Primitive::Boolean(b)) => Value::Primitive(Primitive::Boolean(b.clone())), @@ -47,10 +47,11 @@ pub fn value_to_csv_value(v: &Value) -> Value { Value::Block(_) => Value::Primitive(Primitive::Nothing), _ => Value::Primitive(Primitive::Nothing), } + .tagged(v.tag) } -fn to_string_helper(v: &Value) -> Result { - match v { +fn to_string_helper(v: &Tagged) -> Result { + match &v.item { Value::Primitive(Primitive::Date(d)) => Ok(d.to_string()), Value::Primitive(Primitive::Bytes(b)) => Ok(format!("{}", b)), Value::Primitive(Primitive::Boolean(_)) => Ok(v.as_string()?), @@ -60,7 +61,7 @@ fn to_string_helper(v: &Value) -> Result { Value::Table(_) => return Ok(String::from("[Table]")), Value::Row(_) => return Ok(String::from("[Row]")), Value::Primitive(Primitive::String(s)) => return Ok(s.to_string()), - _ => return Err(ShellError::string("Unexpected value")), + _ => return Err(ShellError::labeled_error("Unexpected value", "", v.tag)), } } @@ -76,7 +77,9 @@ fn merge_descriptors(values: &[Tagged]) -> Vec { ret } -pub fn to_string(v: &Value) -> Result { +pub fn to_string(tagged_value: &Tagged) -> Result { + let v = &tagged_value.item; + match v { Value::Row(o) => { let mut wtr = WriterBuilder::new().from_writer(vec![]); @@ -92,11 +95,20 @@ pub fn to_string(v: &Value) -> Result { wtr.write_record(fields).expect("can not write."); wtr.write_record(values).expect("can not write."); - return Ok(String::from_utf8( - wtr.into_inner() - .map_err(|_| ShellError::string("Could not convert record"))?, - ) - .map_err(|_| ShellError::string("Could not convert record"))?); + return Ok(String::from_utf8(wtr.into_inner().map_err(|_| { + ShellError::labeled_error( + "Could not convert record", + "original value", + tagged_value.tag, + ) + })?) + .map_err(|_| { + ShellError::labeled_error( + "Could not convert record", + "original value", + tagged_value.tag, + ) + })?); } Value::Table(list) => { let mut wtr = WriterBuilder::new().from_writer(vec![]); @@ -120,13 +132,22 @@ pub fn to_string(v: &Value) -> Result { wtr.write_record(&row).expect("can not write"); } - return Ok(String::from_utf8( - wtr.into_inner() - .map_err(|_| ShellError::string("Could not convert record"))?, - ) - .map_err(|_| ShellError::string("Could not convert record"))?); + return Ok(String::from_utf8(wtr.into_inner().map_err(|_| { + ShellError::labeled_error( + "Could not convert record", + "original value", + tagged_value.tag, + ) + })?) + .map_err(|_| { + ShellError::labeled_error( + "Could not convert record", + "original value", + tagged_value.tag, + ) + })?); } - _ => return to_string_helper(&v), + _ => return to_string_helper(tagged_value), } } @@ -148,7 +169,7 @@ fn to_csv( }; for value in to_process_input { - match to_string(&value_to_csv_value(&value.item)) { + match to_string(&value_to_csv_value(&value)) { Ok(x) => { let converted = if headerless { x.lines().skip(1).collect() diff --git a/src/commands/to_tsv.rs b/src/commands/to_tsv.rs index 4edc26face..7127a3195b 100644 --- a/src/commands/to_tsv.rs +++ b/src/commands/to_tsv.rs @@ -32,7 +32,9 @@ impl WholeStreamCommand for ToTSV { } } -pub fn value_to_tsv_value(v: &Value) -> Value { +pub fn value_to_tsv_value(tagged_value: &Tagged) -> Tagged { + let v = &tagged_value.item; + match v { Value::Primitive(Primitive::String(s)) => Value::Primitive(Primitive::String(s.clone())), Value::Primitive(Primitive::Nothing) => Value::Primitive(Primitive::Nothing), @@ -47,20 +49,28 @@ pub fn value_to_tsv_value(v: &Value) -> Value { Value::Block(_) => Value::Primitive(Primitive::Nothing), _ => Value::Primitive(Primitive::Nothing), } + .tagged(tagged_value.tag) } -fn to_string_helper(v: &Value) -> Result { +fn to_string_helper(tagged_value: &Tagged) -> Result { + let v = &tagged_value.item; match v { Value::Primitive(Primitive::Date(d)) => Ok(d.to_string()), Value::Primitive(Primitive::Bytes(b)) => Ok(format!("{}", b)), - Value::Primitive(Primitive::Boolean(_)) => Ok(v.as_string()?), - Value::Primitive(Primitive::Decimal(_)) => Ok(v.as_string()?), - Value::Primitive(Primitive::Int(_)) => Ok(v.as_string()?), - Value::Primitive(Primitive::Path(_)) => Ok(v.as_string()?), + Value::Primitive(Primitive::Boolean(_)) => Ok(tagged_value.as_string()?), + Value::Primitive(Primitive::Decimal(_)) => Ok(tagged_value.as_string()?), + Value::Primitive(Primitive::Int(_)) => Ok(tagged_value.as_string()?), + Value::Primitive(Primitive::Path(_)) => Ok(tagged_value.as_string()?), Value::Table(_) => return Ok(String::from("[table]")), Value::Row(_) => return Ok(String::from("[row]")), Value::Primitive(Primitive::String(s)) => return Ok(s.to_string()), - _ => return Err(ShellError::string("Unexpected value")), + _ => { + return Err(ShellError::labeled_error( + "Unexpected value", + "original value", + tagged_value.tag, + )) + } } } @@ -76,7 +86,9 @@ fn merge_descriptors(values: &[Tagged]) -> Vec { ret } -pub fn to_string(v: &Value) -> Result { +pub fn to_string(tagged_value: &Tagged) -> Result { + let v = &tagged_value.item; + match v { Value::Row(o) => { let mut wtr = WriterBuilder::new().delimiter(b'\t').from_writer(vec![]); @@ -91,11 +103,20 @@ pub fn to_string(v: &Value) -> Result { wtr.write_record(fields).expect("can not write."); wtr.write_record(values).expect("can not write."); - return Ok(String::from_utf8( - wtr.into_inner() - .map_err(|_| ShellError::string("Could not convert record"))?, - ) - .map_err(|_| ShellError::string("Could not convert record"))?); + return Ok(String::from_utf8(wtr.into_inner().map_err(|_| { + ShellError::labeled_error( + "Could not convert record", + "original value", + tagged_value.tag, + ) + })?) + .map_err(|_| { + ShellError::labeled_error( + "Could not convert record", + "original value", + tagged_value.tag, + ) + })?); } Value::Table(list) => { let mut wtr = WriterBuilder::new().delimiter(b'\t').from_writer(vec![]); @@ -119,13 +140,22 @@ pub fn to_string(v: &Value) -> Result { wtr.write_record(&row).expect("can not write"); } - return Ok(String::from_utf8( - wtr.into_inner() - .map_err(|_| ShellError::string("Could not convert record"))?, - ) - .map_err(|_| ShellError::string("Could not convert record"))?); + return Ok(String::from_utf8(wtr.into_inner().map_err(|_| { + ShellError::labeled_error( + "Could not convert record", + "original value", + tagged_value.tag, + ) + })?) + .map_err(|_| { + ShellError::labeled_error( + "Could not convert record", + "original value", + tagged_value.tag, + ) + })?); } - _ => return to_string_helper(&v), + _ => return to_string_helper(tagged_value), } } @@ -147,7 +177,7 @@ fn to_tsv( }; for value in to_process_input { - match to_string(&value_to_tsv_value(&value.item)) { + match to_string(&value_to_tsv_value(&value)) { Ok(x) => { let converted = if headerless { x.lines().skip(1).collect() diff --git a/src/context.rs b/src/context.rs index 6c55aff5c4..a090898328 100644 --- a/src/context.rs +++ b/src/context.rs @@ -7,7 +7,7 @@ use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::error::Error; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use uuid::Uuid; #[derive(Clone, Debug, Serialize, Deserialize)] @@ -77,7 +77,7 @@ impl CommandRegistry { #[derive(Clone)] pub struct Context { registry: CommandRegistry, - pub(crate) source_map: SourceMap, + pub(crate) source_map: Arc>, host: Arc>, pub(crate) shell_manager: ShellManager, } @@ -99,7 +99,7 @@ impl Context { let registry = CommandRegistry::new(); Ok(Context { registry: registry.clone(), - source_map: SourceMap::new(), + source_map: Arc::new(Mutex::new(SourceMap::new())), host: Arc::new(Mutex::new(crate::env::host::BasicHost)), shell_manager: ShellManager::basic(registry)?, }) @@ -118,7 +118,9 @@ impl Context { } pub fn add_anchor_location(&mut self, uuid: Uuid, anchor_location: AnchorLocation) { - self.source_map.insert(uuid, anchor_location); + let mut source_map = self.source_map.lock().unwrap(); + + source_map.insert(uuid, anchor_location); } pub(crate) fn get_command(&self, name: &str) -> Option> { diff --git a/src/data/base.rs b/src/data/base.rs index 176560137f..735196c97f 100644 --- a/src/data/base.rs +++ b/src/data/base.rs @@ -298,7 +298,7 @@ impl fmt::Debug for ValueDebug<'_> { } impl Tagged { - pub(crate) fn tagged_type_name(&self) -> Tagged { + pub fn tagged_type_name(&self) -> Tagged { let name = self.type_name(); Tagged::from_item(name, self.tag()) } @@ -424,10 +424,27 @@ impl Tagged { Ok(out.tagged(self.tag)) } + + pub(crate) fn as_string(&self) -> Result { + match &self.item { + Value::Primitive(Primitive::String(s)) => Ok(s.clone()), + Value::Primitive(Primitive::Boolean(x)) => Ok(format!("{}", x)), + Value::Primitive(Primitive::Decimal(x)) => Ok(format!("{}", x)), + Value::Primitive(Primitive::Int(x)) => Ok(format!("{}", x)), + Value::Primitive(Primitive::Bytes(x)) => Ok(format!("{}", x)), + Value::Primitive(Primitive::Path(x)) => Ok(format!("{}", x.display())), + // TODO: this should definitely be more general with better errors + other => Err(ShellError::labeled_error( + "Expected string", + other.type_name(), + self.tag, + )), + } + } } impl Value { - pub(crate) fn type_name(&self) -> String { + pub fn type_name(&self) -> String { match self { Value::Primitive(p) => p.type_name(), Value::Row(_) => format!("row"), @@ -738,22 +755,6 @@ impl Value { } } - pub(crate) fn as_string(&self) -> Result { - match self { - Value::Primitive(Primitive::String(s)) => Ok(s.clone()), - Value::Primitive(Primitive::Boolean(x)) => Ok(format!("{}", x)), - Value::Primitive(Primitive::Decimal(x)) => Ok(format!("{}", x)), - Value::Primitive(Primitive::Int(x)) => Ok(format!("{}", x)), - Value::Primitive(Primitive::Bytes(x)) => Ok(format!("{}", x)), - Value::Primitive(Primitive::Path(x)) => Ok(format!("{}", x.display())), - // TODO: this should definitely be more general with better errors - other => Err(ShellError::string(format!( - "Expected string, got {:?}", - other - ))), - } - } - pub(crate) fn is_true(&self) -> bool { match self { Value::Primitive(Primitive::Boolean(true)) => true, @@ -806,9 +807,14 @@ impl Value { Value::Primitive(Primitive::Date(s.into())) } - pub fn date_from_str(s: &str) -> Result { - let date = DateTime::parse_from_rfc3339(s) - .map_err(|err| ShellError::string(&format!("Date parse error: {}", err)))?; + pub fn date_from_str(s: Tagged<&str>) -> Result { + let date = DateTime::parse_from_rfc3339(s.item).map_err(|err| { + ShellError::labeled_error( + &format!("Date parse error: {}", err), + "original value", + s.tag, + ) + })?; let date = date.with_timezone(&chrono::offset::Utc); diff --git a/src/data/config.rs b/src/data/config.rs index 1cb4533d8e..657287d2f2 100644 --- a/src/data/config.rs +++ b/src/data/config.rs @@ -51,8 +51,9 @@ pub fn user_data() -> Result { } pub fn app_path(app_data_type: AppDataType, display: &str) -> Result { - let path = app_root(app_data_type, &APP_INFO) - .map_err(|err| ShellError::string(&format!("Couldn't open {} path:\n{}", display, err)))?; + let path = app_root(app_data_type, &APP_INFO).map_err(|err| { + ShellError::untagged_runtime_error(&format!("Couldn't open {} path:\n{}", display, err)) + })?; Ok(path) } @@ -75,10 +76,21 @@ pub fn read( let tag = tag.into(); let contents = fs::read_to_string(filename) .map(|v| v.tagged(tag)) - .map_err(|err| ShellError::string(&format!("Couldn't read config file:\n{}", err)))?; + .map_err(|err| { + ShellError::labeled_error( + &format!("Couldn't read config file:\n{}", err), + "file name", + tag, + ) + })?; - let parsed: toml::Value = toml::from_str(&contents) - .map_err(|err| ShellError::string(&format!("Couldn't parse config file:\n{}", err)))?; + let parsed: toml::Value = toml::from_str(&contents).map_err(|err| { + ShellError::labeled_error( + &format!("Couldn't parse config file:\n{}", err), + "file name", + tag, + ) + })?; let value = convert_toml_value_to_nu_value(&parsed, tag); let tag = value.tag(); diff --git a/src/data/meta.rs b/src/data/meta.rs index b66b009cc2..08125359e4 100644 --- a/src/data/meta.rs +++ b/src/data/meta.rs @@ -240,6 +240,16 @@ impl Tag { } } + pub fn for_char(pos: usize, anchor: Uuid) -> Tag { + Tag { + anchor, + span: Span { + start: pos, + end: pos + 1, + }, + } + } + pub fn unknown_span(anchor: Uuid) -> Tag { Tag { anchor, @@ -267,6 +277,24 @@ impl Tag { } } + pub fn until_option(&self, other: Option>) -> Tag { + match other { + Some(other) => { + let other = other.into(); + debug_assert!( + self.anchor == other.anchor, + "Can only merge two tags with the same anchor" + ); + + Tag { + span: Span::new(self.span.start, other.span.end), + anchor: self.anchor, + } + } + None => *self, + } + } + pub fn slice<'a>(&self, source: &'a str) -> &'a str { self.span.slice(source) } @@ -284,6 +312,7 @@ impl Tag { } } +#[allow(unused)] pub fn tag_for_tagged_list(mut iter: impl Iterator) -> Tag { let first = iter.next(); diff --git a/src/errors.rs b/src/errors.rs index a070f6f54e..2d42552250 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -20,6 +20,14 @@ impl Description { Description::Synthetic(s) => Err(s), } } + + #[allow(unused)] + fn tag(&self) -> Tag { + match self { + Description::Source(tagged) => tagged.tag, + Description::Synthetic(_) => Tag::unknown(), + } + } } #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)] @@ -36,6 +44,13 @@ pub struct ShellError { cause: Option>, } +impl ShellError { + #[allow(unused)] + pub(crate) fn tag(&self) -> Option { + self.error.tag() + } +} + impl ToDebug for ShellError { fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { self.error.fmt_debug(f, source) @@ -47,12 +62,12 @@ impl serde::de::Error for ShellError { where T: std::fmt::Display, { - ShellError::string(msg.to_string()) + ShellError::untagged_runtime_error(msg.to_string()) } } impl ShellError { - pub(crate) fn type_error( + pub fn type_error( expected: impl Into, actual: Tagged>, ) -> ShellError { @@ -63,6 +78,13 @@ impl ShellError { .start() } + pub fn untagged_runtime_error(error: impl Into) -> ShellError { + ProximateShellError::UntaggedRuntimeError { + reason: error.into(), + } + .start() + } + pub(crate) fn unexpected_eof(expected: impl Into, tag: Tag) -> ShellError { ProximateShellError::UnexpectedEof { expected: expected.into(), @@ -174,9 +196,6 @@ impl ShellError { pub(crate) fn to_diagnostic(self) -> Diagnostic { match self.error { - ProximateShellError::String(StringError { title, .. }) => { - Diagnostic::new(Severity::Error, title) - } ProximateShellError::InvalidCommand { command } => { Diagnostic::new(Severity::Error, "Invalid command") .with_label(Label::new_primary(command)) @@ -286,7 +305,7 @@ impl ShellError { } => Diagnostic::new(Severity::Error, "Syntax Error") .with_label(Label::new_primary(tag).with_message(item)), - ProximateShellError::MissingProperty { subpath, expr } => { + ProximateShellError::MissingProperty { subpath, expr, .. } => { let subpath = subpath.into_label(); let expr = expr.into_label(); @@ -310,6 +329,8 @@ impl ShellError { .with_label(Label::new_primary(left.tag()).with_message(left.item)) .with_label(Label::new_secondary(right.tag()).with_message(right.item)) } + + ProximateShellError::UntaggedRuntimeError { reason } => Diagnostic::new(Severity::Error, format!("Error: {}", reason)) } } @@ -343,20 +364,16 @@ impl ShellError { ) } - pub fn string(title: impl Into) -> ShellError { - ProximateShellError::String(StringError::new(title.into(), Value::nothing())).start() - } - pub(crate) fn unimplemented(title: impl Into) -> ShellError { - ShellError::string(&format!("Unimplemented: {}", title.into())) + ShellError::untagged_runtime_error(&format!("Unimplemented: {}", title.into())) } pub(crate) fn unexpected(title: impl Into) -> ShellError { - ShellError::string(&format!("Unexpected: {}", title.into())) + ShellError::untagged_runtime_error(&format!("Unexpected: {}", title.into())) } pub(crate) fn unreachable(title: impl Into) -> ShellError { - ShellError::string(&format!("BUG: Unreachable: {}", title.into())) + ShellError::untagged_runtime_error(&format!("BUG: Unreachable: {}", title.into())) } } @@ -401,7 +418,6 @@ impl ExpectedRange { #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)] pub enum ProximateShellError { - String(StringError), SyntaxError { problem: Tagged, }, @@ -419,6 +435,7 @@ pub enum ProximateShellError { MissingProperty { subpath: Description, expr: Description, + tag: Tag, }, MissingValue { tag: Option, @@ -439,6 +456,9 @@ pub enum ProximateShellError { left: Tagged, right: Tagged, }, + UntaggedRuntimeError { + reason: String, + }, } impl ProximateShellError { @@ -448,6 +468,22 @@ impl ProximateShellError { error: self, } } + + pub(crate) fn tag(&self) -> Option { + Some(match self { + ProximateShellError::SyntaxError { problem } => problem.tag(), + ProximateShellError::UnexpectedEof { tag, .. } => *tag, + ProximateShellError::InvalidCommand { command } => *command, + ProximateShellError::TypeError { actual, .. } => actual.tag, + ProximateShellError::MissingProperty { tag, .. } => *tag, + ProximateShellError::MissingValue { tag, .. } => return *tag, + ProximateShellError::ArgumentError { tag, .. } => *tag, + ProximateShellError::RangeError { actual_kind, .. } => actual_kind.tag, + ProximateShellError::Diagnostic(..) => return None, + ProximateShellError::UntaggedRuntimeError { .. } => return None, + ProximateShellError::CoerceError { left, right } => left.tag.until(right.tag), + }) + } } impl ToDebug for ProximateShellError { @@ -491,7 +527,6 @@ pub struct StringError { impl std::fmt::Display for ShellError { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match &self.error { - ProximateShellError::String(s) => write!(f, "{}", &s.title), ProximateShellError::MissingValue { .. } => write!(f, "MissingValue"), ProximateShellError::InvalidCommand { .. } => write!(f, "InvalidCommand"), ProximateShellError::TypeError { .. } => write!(f, "TypeError"), @@ -502,6 +537,7 @@ impl std::fmt::Display for ShellError { ProximateShellError::ArgumentError { .. } => write!(f, "ArgumentError"), ProximateShellError::Diagnostic(_) => write!(f, ""), ProximateShellError::CoerceError { .. } => write!(f, "CoerceError"), + ProximateShellError::UntaggedRuntimeError { .. } => write!(f, "UntaggedRuntimeError"), } } } @@ -510,71 +546,43 @@ impl std::error::Error for ShellError {} impl std::convert::From> for ShellError { fn from(input: Box) -> ShellError { - ProximateShellError::String(StringError { - title: format!("{}", input), - error: Value::nothing(), - }) - .start() + ShellError::untagged_runtime_error(format!("{}", input)) } } impl std::convert::From for ShellError { fn from(input: std::io::Error) -> ShellError { - ProximateShellError::String(StringError { - title: format!("{}", input), - error: Value::nothing(), - }) - .start() + ShellError::untagged_runtime_error(format!("{}", input)) } } impl std::convert::From for ShellError { fn from(input: subprocess::PopenError) -> ShellError { - ProximateShellError::String(StringError { - title: format!("{}", input), - error: Value::nothing(), - }) - .start() + ShellError::untagged_runtime_error(format!("{}", input)) } } impl std::convert::From for ShellError { fn from(input: serde_yaml::Error) -> ShellError { - ProximateShellError::String(StringError { - title: format!("{:?}", input), - error: Value::nothing(), - }) - .start() + ShellError::untagged_runtime_error(format!("{:?}", input)) } } impl std::convert::From for ShellError { fn from(input: toml::ser::Error) -> ShellError { - ProximateShellError::String(StringError { - title: format!("{:?}", input), - error: Value::nothing(), - }) - .start() + ShellError::untagged_runtime_error(format!("{:?}", input)) } } impl std::convert::From for ShellError { fn from(input: serde_json::Error) -> ShellError { - ProximateShellError::String(StringError { - title: format!("{:?}", input), - error: Value::nothing(), - }) - .start() + ShellError::untagged_runtime_error(format!("{:?}", input)) } } impl std::convert::From> for ShellError { fn from(input: Box) -> ShellError { - ProximateShellError::String(StringError { - title: format!("{:?}", input), - error: Value::nothing(), - }) - .start() + ShellError::untagged_runtime_error(format!("{:?}", input)) } } diff --git a/src/parser.rs b/src/parser.rs index 5fcfaaa27e..3fd853c85c 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -7,18 +7,18 @@ pub(crate) mod registry; use crate::errors::ShellError; pub(crate) use deserializer::ConfigDeserializer; +pub(crate) use hir::syntax_shape::flat_shape::FlatShape; pub(crate) use hir::TokensIterator; pub(crate) use parse::call_node::CallNode; pub(crate) use parse::files::Files; -pub(crate) use parse::flag::Flag; +pub(crate) use parse::flag::{Flag, FlagKind}; pub(crate) use parse::operator::Operator; pub(crate) use parse::parser::{nom_input, pipeline}; pub(crate) use parse::pipeline::{Pipeline, PipelineElement}; pub(crate) use parse::text::Text; pub(crate) use parse::token_tree::{DelimitedNode, Delimiter, TokenNode}; -pub(crate) use parse::tokens::{RawToken, Token}; +pub(crate) use parse::tokens::{RawNumber, RawToken}; pub(crate) use parse::unit::Unit; -pub(crate) use parse_command::parse_command_tail; pub(crate) use registry::CommandRegistry; pub fn parse(input: &str, anchor: uuid::Uuid) -> Result { diff --git a/src/parser/hir/expand_external_tokens.rs b/src/parser/hir/expand_external_tokens.rs index 30a2a90aaf..238cb4b01b 100644 --- a/src/parser/hir/expand_external_tokens.rs +++ b/src/parser/hir/expand_external_tokens.rs @@ -1,5 +1,11 @@ use crate::errors::ShellError; -use crate::parser::{TokenNode, TokensIterator}; +use crate::parser::{ + hir::syntax_shape::{ + color_syntax, expand_atom, AtomicToken, ColorSyntax, ExpandContext, ExpansionRule, + MaybeSpaceShape, + }, + FlatShape, TokenNode, TokensIterator, +}; use crate::{Tag, Tagged, Text}; pub fn expand_external_tokens( @@ -19,6 +25,34 @@ pub fn expand_external_tokens( Ok(out) } +#[derive(Debug, Copy, Clone)] +pub struct ExternalTokensShape; + +impl ColorSyntax for ExternalTokensShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Self::Info { + loop { + // Allow a space + color_syntax(&MaybeSpaceShape, token_nodes, context, shapes); + + // Process an external expression. External expressions are mostly words, with a + // few exceptions (like $variables and path expansion rules) + match color_syntax(&ExternalExpression, token_nodes, context, shapes).1 { + ExternalExpressionResult::Eof => break, + ExternalExpressionResult::Processed => continue, + } + } + } +} + pub fn expand_next_expression( token_nodes: &mut TokensIterator<'_>, ) -> Result, ShellError> { @@ -48,16 +82,15 @@ pub fn expand_next_expression( fn triage_external_head(node: &TokenNode) -> Result { Ok(match node { TokenNode::Token(token) => token.tag(), - TokenNode::Call(_call) => unimplemented!(), - TokenNode::Nodes(_nodes) => unimplemented!(), - TokenNode::Delimited(_delimited) => unimplemented!(), - TokenNode::Pipeline(_pipeline) => unimplemented!(), + TokenNode::Call(_call) => unimplemented!("TODO: OMG"), + TokenNode::Nodes(_nodes) => unimplemented!("TODO: OMG"), + TokenNode::Delimited(_delimited) => unimplemented!("TODO: OMG"), + TokenNode::Pipeline(_pipeline) => unimplemented!("TODO: OMG"), TokenNode::Flag(flag) => flag.tag(), - TokenNode::Member(member) => *member, TokenNode::Whitespace(_whitespace) => { unreachable!("This function should be called after next_non_ws()") } - TokenNode::Error(_error) => unimplemented!(), + TokenNode::Error(_error) => unimplemented!("TODO: OMG"), }) } @@ -73,7 +106,7 @@ fn triage_continuation<'a, 'b>( match &node { node if node.is_whitespace() => return Ok(None), - TokenNode::Token(..) | TokenNode::Flag(..) | TokenNode::Member(..) => {} + TokenNode::Token(..) | TokenNode::Flag(..) => {} TokenNode::Call(..) => unimplemented!("call"), TokenNode::Nodes(..) => unimplemented!("nodes"), TokenNode::Delimited(..) => unimplemented!("delimited"), @@ -85,3 +118,42 @@ fn triage_continuation<'a, 'b>( peeked.commit(); Ok(Some(node.tag())) } + +#[must_use] +enum ExternalExpressionResult { + Eof, + Processed, +} + +#[derive(Debug, Copy, Clone)] +struct ExternalExpression; + +impl ColorSyntax for ExternalExpression { + type Info = ExternalExpressionResult; + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> ExternalExpressionResult { + let atom = match expand_atom( + token_nodes, + "external word", + context, + ExpansionRule::permissive(), + ) { + Err(_) => unreachable!("TODO: separate infallible expand_atom"), + Ok(Tagged { + item: AtomicToken::Eof { .. }, + .. + }) => return ExternalExpressionResult::Eof, + Ok(atom) => atom, + }; + + atom.color_tokens(shapes); + return ExternalExpressionResult::Processed; + } +} diff --git a/src/parser/hir/syntax_shape.rs b/src/parser/hir/syntax_shape.rs index 5dcbd0fb76..1a140d86bd 100644 --- a/src/parser/hir/syntax_shape.rs +++ b/src/parser/hir/syntax_shape.rs @@ -1,34 +1,45 @@ mod block; mod expression; +pub(crate) mod flat_shape; use crate::cli::external_command; -use crate::commands::{classified::InternalCommand, ClassifiedCommand, Command}; +use crate::commands::{ + classified::{ClassifiedPipeline, InternalCommand}, + ClassifiedCommand, Command, +}; +use crate::parser::hir::expand_external_tokens::ExternalTokensShape; use crate::parser::hir::syntax_shape::block::AnyBlockShape; use crate::parser::hir::tokens_iterator::Peeked; -use crate::parser::parse_command::parse_command_tail; +use crate::parser::parse_command::{parse_command_tail, CommandTailShape}; +use crate::parser::PipelineElement; use crate::parser::{ hir, hir::{debug_tokens, TokensIterator}, - Operator, RawToken, TokenNode, + Operator, Pipeline, RawToken, TokenNode, }; use crate::prelude::*; use derive_new::new; use getset::Getters; -use log::trace; +use log::{self, log_enabled, trace}; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; +pub(crate) use self::expression::atom::{expand_atom, AtomicToken, ExpansionRule}; +pub(crate) use self::expression::delimited::{ + color_delimited_square, expand_delimited_square, DelimitedShape, +}; pub(crate) use self::expression::file_path::FilePathShape; -pub(crate) use self::expression::list::ExpressionListShape; +pub(crate) use self::expression::list::{BackoffColoringMode, ExpressionListShape}; pub(crate) use self::expression::number::{IntShape, NumberShape}; -pub(crate) use self::expression::pattern::PatternShape; +pub(crate) use self::expression::pattern::{BarePatternShape, PatternShape}; pub(crate) use self::expression::string::StringShape; pub(crate) use self::expression::unit::UnitShape; pub(crate) use self::expression::variable_path::{ - ColumnPathShape, DotShape, ExpressionContinuation, ExpressionContinuationShape, MemberShape, - PathTailShape, VariablePathShape, + ColorableDotShape, ColumnPathShape, DotShape, ExpressionContinuation, + ExpressionContinuationShape, MemberShape, PathTailShape, VariablePathShape, }; pub(crate) use self::expression::{continue_expression, AnyExpressionShape}; +pub(crate) use self::flat_shape::FlatShape; #[derive(Debug, Copy, Clone, Serialize, Deserialize)] pub enum SyntaxShape { @@ -41,9 +52,56 @@ pub enum SyntaxShape { Int, Path, Pattern, - Binary, Block, - Boolean, +} + +impl FallibleColorSyntax for SyntaxShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + match self { + SyntaxShape::Any => { + color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes) + } + SyntaxShape::List => { + color_syntax(&ExpressionListShape, token_nodes, context, shapes); + Ok(()) + } + SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context, shapes), + SyntaxShape::String => color_fallible_syntax_with( + &StringShape, + &FlatShape::String, + token_nodes, + context, + shapes, + ), + SyntaxShape::Member => { + color_fallible_syntax(&MemberShape, token_nodes, context, shapes) + } + SyntaxShape::ColumnPath => { + color_fallible_syntax(&ColumnPathShape, token_nodes, context, shapes) + } + SyntaxShape::Number => { + color_fallible_syntax(&NumberShape, token_nodes, context, shapes) + } + SyntaxShape::Path => { + color_fallible_syntax(&FilePathShape, token_nodes, context, shapes) + } + SyntaxShape::Pattern => { + color_fallible_syntax(&PatternShape, token_nodes, context, shapes) + } + SyntaxShape::Block => { + color_fallible_syntax(&AnyBlockShape, token_nodes, context, shapes) + } + } + } } impl ExpandExpression for SyntaxShape { @@ -73,9 +131,7 @@ impl ExpandExpression for SyntaxShape { SyntaxShape::Number => expand_expr(&NumberShape, token_nodes, context), SyntaxShape::Path => expand_expr(&FilePathShape, token_nodes, context), SyntaxShape::Pattern => expand_expr(&PatternShape, token_nodes, context), - SyntaxShape::Binary => Err(ShellError::unimplemented("SyntaxShape:Binary")), SyntaxShape::Block => expand_expr(&AnyBlockShape, token_nodes, context), - SyntaxShape::Boolean => Err(ShellError::unimplemented("SyntaxShape:Boolean")), } } } @@ -92,9 +148,7 @@ impl std::fmt::Display for SyntaxShape { SyntaxShape::Number => write!(f, "Number"), SyntaxShape::Path => write!(f, "Path"), SyntaxShape::Pattern => write!(f, "Pattern"), - SyntaxShape::Binary => write!(f, "Binary"), SyntaxShape::Block => write!(f, "Block"), - SyntaxShape::Boolean => write!(f, "Boolean"), } } } @@ -148,6 +202,50 @@ pub trait ExpandExpression: std::fmt::Debug + Copy { ) -> Result; } +pub trait FallibleColorSyntax: std::fmt::Debug + Copy { + type Info; + type Input; + + fn color_syntax<'a, 'b>( + &self, + input: &Self::Input, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result; +} + +pub trait ColorSyntax: std::fmt::Debug + Copy { + type Info; + type Input; + + fn color_syntax<'a, 'b>( + &self, + input: &Self::Input, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Self::Info; +} + +// impl ColorSyntax for T +// where +// T: FallibleColorSyntax, +// { +// type Info = Result; +// type Input = T::Input; + +// fn color_syntax<'a, 'b>( +// &self, +// input: &Self::Input, +// token_nodes: &'b mut TokensIterator<'a>, +// context: &ExpandContext, +// shapes: &mut Vec>, +// ) -> Result { +// FallibleColorSyntax::color_syntax(self, input, token_nodes, context, shapes) +// } +// } + pub(crate) trait ExpandSyntax: std::fmt::Debug + Copy { type Output: std::fmt::Debug; @@ -180,6 +278,130 @@ pub(crate) fn expand_syntax<'a, 'b, T: ExpandSyntax>( } } +pub fn color_syntax<'a, 'b, T: ColorSyntax, U>( + shape: &T, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, +) -> ((), U) { + trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); + + let len = shapes.len(); + let result = shape.color_syntax(&(), token_nodes, context, shapes); + + trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes, context.source)); + + if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { + trace!(target: "nu::color_syntax", "after {}", std::any::type_name::()); + + if len < shapes.len() { + for i in len..(shapes.len()) { + trace!(target: "nu::color_syntax", "new shape :: {:?}", shapes[i]); + } + } else { + trace!(target: "nu::color_syntax", "no new shapes"); + } + } + + ((), result) +} + +pub fn color_fallible_syntax<'a, 'b, T: FallibleColorSyntax, U>( + shape: &T, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, +) -> Result { + trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); + + if token_nodes.at_end() { + trace!(target: "nu::color_syntax", "at eof"); + return Err(ShellError::unexpected_eof("coloring", Tag::unknown())); + } + + let len = shapes.len(); + let result = shape.color_syntax(&(), token_nodes, context, shapes); + + trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes, context.source)); + + if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { + trace!(target: "nu::color_syntax", "after {}", std::any::type_name::()); + + if len < shapes.len() { + for i in len..(shapes.len()) { + trace!(target: "nu::color_syntax", "new shape :: {:?}", shapes[i]); + } + } else { + trace!(target: "nu::color_syntax", "no new shapes"); + } + } + + result +} + +pub fn color_syntax_with<'a, 'b, T: ColorSyntax, U, I>( + shape: &T, + input: &I, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, +) -> ((), U) { + trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); + + let len = shapes.len(); + let result = shape.color_syntax(input, token_nodes, context, shapes); + + trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes, context.source)); + + if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { + trace!(target: "nu::color_syntax", "after {}", std::any::type_name::()); + + if len < shapes.len() { + for i in len..(shapes.len()) { + trace!(target: "nu::color_syntax", "new shape :: {:?}", shapes[i]); + } + } else { + trace!(target: "nu::color_syntax", "no new shapes"); + } + } + + ((), result) +} + +pub fn color_fallible_syntax_with<'a, 'b, T: FallibleColorSyntax, U, I>( + shape: &T, + input: &I, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, +) -> Result { + trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); + + if token_nodes.at_end() { + trace!(target: "nu::color_syntax", "at eof"); + return Err(ShellError::unexpected_eof("coloring", Tag::unknown())); + } + + let len = shapes.len(); + let result = shape.color_syntax(input, token_nodes, context, shapes); + + trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes, context.source)); + + if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { + trace!(target: "nu::color_syntax", "after {}", std::any::type_name::()); + + if len < shapes.len() { + for i in len..(shapes.len()) { + trace!(target: "nu::color_syntax", "new shape :: {:?}", shapes[i]); + } + } else { + trace!(target: "nu::color_syntax", "no new shapes"); + } + } + + result +} + pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>( shape: &T, token_nodes: &'b mut TokensIterator<'a>, @@ -314,6 +536,33 @@ impl ExpandSyntax for BarePathShape { #[derive(Debug, Copy, Clone)] pub struct BareShape; +impl FallibleColorSyntax for BareShape { + type Info = (); + type Input = FlatShape; + + fn color_syntax<'a, 'b>( + &self, + input: &FlatShape, + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + token_nodes.peek_any_token(|token| match token { + // If it's a bare token, color it + TokenNode::Token(Tagged { + item: RawToken::Bare, + tag, + }) => { + shapes.push((*input).tagged(tag)); + Ok(()) + } + + // otherwise, fail + other => Err(ShellError::type_error("word", other.tagged_type_name())), + }) + } +} + impl ExpandSyntax for BareShape { type Output = Tagged; @@ -383,9 +632,129 @@ impl CommandSignature { } } +#[derive(Debug, Copy, Clone)] +pub struct PipelineShape; + +// The failure mode is if the head of the token stream is not a pipeline +impl FallibleColorSyntax for PipelineShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + // Make sure we're looking at a pipeline + let Pipeline { parts, .. } = token_nodes.peek_any_token(|node| node.as_pipeline())?; + + // Enumerate the pipeline parts + for part in parts { + // If the pipeline part has a prefix `|`, emit a pipe to color + if let Some(pipe) = part.pipe { + shapes.push(FlatShape::Pipe.tagged(pipe)); + } + + // Create a new iterator containing the tokens in the pipeline part to color + let mut token_nodes = TokensIterator::new(&part.tokens.item, part.tag, false); + + color_syntax(&MaybeSpaceShape, &mut token_nodes, context, shapes); + color_syntax(&CommandShape, &mut token_nodes, context, shapes); + } + + Ok(()) + } +} + +impl ExpandSyntax for PipelineShape { + type Output = ClassifiedPipeline; + fn expand_syntax<'a, 'b>( + &self, + iterator: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let source = context.source; + + let peeked = iterator.peek_any().not_eof("pipeline")?; + let pipeline = peeked.node.as_pipeline()?; + peeked.commit(); + + let Pipeline { parts, .. } = pipeline; + + let commands: Result, ShellError> = parts + .iter() + .map(|item| classify_command(&item, context, &source)) + .collect(); + + Ok(ClassifiedPipeline { + commands: commands?, + }) + } +} + +pub enum CommandHeadKind { + External, + Internal(Signature), +} + #[derive(Debug, Copy, Clone)] pub struct CommandHeadShape; +impl FallibleColorSyntax for CommandHeadShape { + type Info = CommandHeadKind; + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result { + // If we don't ultimately find a token, roll back + token_nodes.atomic(|token_nodes| { + // First, take a look at the next token + let atom = expand_atom( + token_nodes, + "command head", + context, + ExpansionRule::permissive(), + )?; + + match atom.item { + // If the head is an explicit external command (^cmd), color it as an external command + AtomicToken::ExternalCommand { command } => { + shapes.push(FlatShape::ExternalCommand.tagged(command)); + Ok(CommandHeadKind::External) + } + + // If the head is a word, it depends on whether it matches a registered internal command + AtomicToken::Word { text } => { + let name = text.slice(context.source); + + if context.registry.has(name) { + // If the registry has the command, color it as an internal command + shapes.push(FlatShape::InternalCommand.tagged(text)); + let command = context.registry.expect_command(name); + Ok(CommandHeadKind::Internal(command.signature())) + } else { + // Otherwise, color it as an external command + shapes.push(FlatShape::ExternalCommand.tagged(text)); + Ok(CommandHeadKind::External) + } + } + + // Otherwise, we're not actually looking at a command + _ => Err(ShellError::syntax_error( + "No command at the head".tagged(atom.tag), + )), + } + }) + } +} + impl ExpandSyntax for CommandHeadShape { type Output = CommandSignature; @@ -395,7 +764,7 @@ impl ExpandSyntax for CommandHeadShape { context: &ExpandContext, ) -> Result { let node = - parse_single_node_skipping_ws(token_nodes, "command head1", |token, token_tag| { + parse_single_node_skipping_ws(token_nodes, "command head1", |token, token_tag, _| { Ok(match token { RawToken::ExternalCommand(tag) => CommandSignature::LiteralExternal { outer: token_tag, @@ -488,6 +857,44 @@ impl ExpandSyntax for ClassifiedCommandShape { #[derive(Debug, Copy, Clone)] pub struct InternalCommandHeadShape; +impl FallibleColorSyntax for InternalCommandHeadShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let peeked_head = token_nodes.peek_non_ws().not_eof("command head4"); + + let peeked_head = match peeked_head { + Err(_) => return Ok(()), + Ok(peeked_head) => peeked_head, + }; + + let _expr = match peeked_head.node { + TokenNode::Token(Tagged { + item: RawToken::Bare, + tag, + }) => shapes.push(FlatShape::Word.tagged(tag)), + + TokenNode::Token(Tagged { + item: RawToken::String(_inner_tag), + tag, + }) => shapes.push(FlatShape::String.tagged(tag)), + + _node => shapes.push(FlatShape::Error.tagged(peeked_head.node.tag())), + }; + + peeked_head.commit(); + + Ok(()) + } +} + impl ExpandExpression for InternalCommandHeadShape { fn expand_expr( &self, @@ -523,33 +930,52 @@ impl ExpandExpression for InternalCommandHeadShape { } } +pub(crate) struct SingleError<'token> { + expected: &'static str, + node: &'token Tagged, +} + +impl<'token> SingleError<'token> { + pub(crate) fn error(&self) -> ShellError { + ShellError::type_error(self.expected, self.node.type_name().tagged(self.node.tag)) + } +} + fn parse_single_node<'a, 'b, T>( token_nodes: &'b mut TokensIterator<'a>, expected: &'static str, - callback: impl FnOnce(RawToken, Tag) -> Result, + callback: impl FnOnce(RawToken, Tag, SingleError) -> Result, ) -> Result { - let peeked = token_nodes.peek_any().not_eof(expected)?; + token_nodes.peek_any_token(|node| match node { + TokenNode::Token(token) => callback( + token.item, + token.tag(), + SingleError { + expected, + node: token, + }, + ), - let expr = match peeked.node { - TokenNode::Token(token) => callback(token.item, token.tag())?, - - other => return Err(ShellError::type_error(expected, other.tagged_type_name())), - }; - - peeked.commit(); - - Ok(expr) + other => Err(ShellError::type_error(expected, other.tagged_type_name())), + }) } fn parse_single_node_skipping_ws<'a, 'b, T>( token_nodes: &'b mut TokensIterator<'a>, expected: &'static str, - callback: impl FnOnce(RawToken, Tag) -> Result, + callback: impl FnOnce(RawToken, Tag, SingleError) -> Result, ) -> Result { let peeked = token_nodes.peek_non_ws().not_eof(expected)?; let expr = match peeked.node { - TokenNode::Token(token) => callback(token.item, token.tag())?, + TokenNode::Token(token) => callback( + token.item, + token.tag(), + SingleError { + expected, + node: token, + }, + )?, other => return Err(ShellError::type_error(expected, other.tagged_type_name())), }; @@ -562,6 +988,36 @@ fn parse_single_node_skipping_ws<'a, 'b, T>( #[derive(Debug, Copy, Clone)] pub struct WhitespaceShape; +impl FallibleColorSyntax for WhitespaceShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let peeked = token_nodes.peek_any().not_eof("whitespace"); + + let peeked = match peeked { + Err(_) => return Ok(()), + Ok(peeked) => peeked, + }; + + let _tag = match peeked.node { + TokenNode::Whitespace(tag) => shapes.push(FlatShape::Whitespace.tagged(tag)), + + _other => return Ok(()), + }; + + peeked.commit(); + + Ok(()) + } +} + impl ExpandSyntax for WhitespaceShape { type Output = Tag; @@ -626,6 +1082,65 @@ pub struct MaybeSpacedExpression { inner: T, } +#[derive(Debug, Copy, Clone)] +pub struct MaybeSpaceShape; + +impl ColorSyntax for MaybeSpaceShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + shapes: &mut Vec>, + ) -> Self::Info { + let peeked = token_nodes.peek_any().not_eof("whitespace"); + + let peeked = match peeked { + Err(_) => return, + Ok(peeked) => peeked, + }; + + if let TokenNode::Whitespace(tag) = peeked.node { + peeked.commit(); + shapes.push(FlatShape::Whitespace.tagged(tag)); + } + } +} + +#[derive(Debug, Copy, Clone)] +pub struct SpaceShape; + +impl FallibleColorSyntax for SpaceShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let peeked = token_nodes.peek_any().not_eof("whitespace")?; + + match peeked.node { + TokenNode::Whitespace(tag) => { + peeked.commit(); + shapes.push(FlatShape::Whitespace.tagged(tag)); + Ok(()) + } + + other => Err(ShellError::type_error( + "whitespace", + other.tagged_type_name(), + )), + } + } +} + impl ExpandExpression for MaybeSpacedExpression { fn expand_expr<'a, 'b>( &self, @@ -660,3 +1175,87 @@ fn expand_variable(tag: Tag, token_tag: Tag, source: &Text) -> hir::Expression { hir::Expression::variable(tag, token_tag) } } + +fn classify_command( + command: &Tagged, + context: &ExpandContext, + source: &Text, +) -> Result { + let mut iterator = TokensIterator::new(&command.tokens.item, command.tag, true); + + let head = CommandHeadShape.expand_syntax(&mut iterator, &context)?; + + match &head { + CommandSignature::Expression(_) => Err(ShellError::syntax_error( + "Unexpected expression in command position".tagged(command.tag), + )), + + // If the command starts with `^`, treat it as an external command no matter what + CommandSignature::External(name) => { + let name_str = name.slice(source); + + external_command(&mut iterator, source, name_str.tagged(name)) + } + + CommandSignature::LiteralExternal { outer, inner } => { + let name_str = inner.slice(source); + + external_command(&mut iterator, source, name_str.tagged(outer)) + } + + CommandSignature::Internal(command) => { + let tail = + parse_command_tail(&command.signature(), &context, &mut iterator, command.tag)?; + + let (positional, named) = match tail { + None => (None, None), + Some((positional, named)) => (positional, named), + }; + + let call = hir::Call { + head: Box::new(head.to_expression()), + positional, + named, + }; + + Ok(ClassifiedCommand::Internal(InternalCommand::new( + command.name().to_string(), + command.tag, + call, + ))) + } + } +} + +#[derive(Debug, Copy, Clone)] +pub struct CommandShape; + +impl ColorSyntax for CommandShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) { + let kind = color_fallible_syntax(&CommandHeadShape, token_nodes, context, shapes); + + match kind { + Err(_) => { + // We didn't find a command, so we'll have to fall back to parsing this pipeline part + // as a blob of undifferentiated expressions + color_syntax(&ExpressionListShape, token_nodes, context, shapes); + } + + Ok(CommandHeadKind::External) => { + color_syntax(&ExternalTokensShape, token_nodes, context, shapes); + } + Ok(CommandHeadKind::Internal(signature)) => { + color_syntax_with(&CommandTailShape, &signature, token_nodes, context, shapes); + } + }; + } +} diff --git a/src/parser/hir/syntax_shape/block.rs b/src/parser/hir/syntax_shape/block.rs index a78292b34e..806681691e 100644 --- a/src/parser/hir/syntax_shape/block.rs +++ b/src/parser/hir/syntax_shape/block.rs @@ -2,10 +2,13 @@ use crate::errors::ShellError; use crate::parser::{ hir, hir::syntax_shape::{ - continue_expression, expand_expr, expand_syntax, ExpandContext, ExpandExpression, - ExpressionListShape, PathTailShape, VariablePathShape, + color_fallible_syntax, color_syntax_with, continue_expression, expand_expr, expand_syntax, + DelimitedShape, ExpandContext, ExpandExpression, ExpressionContinuationShape, + ExpressionListShape, FallibleColorSyntax, FlatShape, MemberShape, PathTailShape, + VariablePathShape, }, hir::tokens_iterator::TokensIterator, + parse::token_tree::Delimiter, RawToken, TokenNode, }; use crate::{Tag, Tagged, TaggedItem}; @@ -13,6 +16,49 @@ use crate::{Tag, Tagged, TaggedItem}; #[derive(Debug, Copy, Clone)] pub struct AnyBlockShape; +impl FallibleColorSyntax for AnyBlockShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let block = token_nodes.peek_non_ws().not_eof("block"); + + let block = match block { + Err(_) => return Ok(()), + Ok(block) => block, + }; + + // is it just a block? + let block = block.node.as_block(); + + match block { + // If so, color it as a block + Some((children, tags)) => { + let mut token_nodes = TokensIterator::new(children.item, context.tag, false); + color_syntax_with( + &DelimitedShape, + &(Delimiter::Brace, tags.0, tags.1), + &mut token_nodes, + context, + shapes, + ); + + return Ok(()); + } + _ => {} + } + + // Otherwise, look for a shorthand block. If none found, fail + color_fallible_syntax(&ShorthandBlock, token_nodes, context, shapes) + } +} + impl ExpandExpression for AnyBlockShape { fn expand_expr<'a, 'b>( &self, @@ -25,7 +71,7 @@ impl ExpandExpression for AnyBlockShape { let block = block.node.as_block(); match block { - Some(block) => { + Some((block, _tags)) => { let mut iterator = TokensIterator::new(&block.item, context.tag, false); let exprs = expand_syntax(&ExpressionListShape, &mut iterator, context)?; @@ -42,6 +88,37 @@ impl ExpandExpression for AnyBlockShape { #[derive(Debug, Copy, Clone)] pub struct ShorthandBlock; +impl FallibleColorSyntax for ShorthandBlock { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + // Try to find a shorthand head. If none found, fail + color_fallible_syntax(&ShorthandPath, token_nodes, context, shapes)?; + + loop { + // Check to see whether there's any continuation after the head expression + let result = + color_fallible_syntax(&ExpressionContinuationShape, token_nodes, context, shapes); + + match result { + // if no continuation was found, we're done + Err(_) => break, + // if a continuation was found, look for another one + Ok(_) => continue, + } + } + + Ok(()) + } +} + impl ExpandExpression for ShorthandBlock { fn expand_expr<'a, 'b>( &self, @@ -62,6 +139,50 @@ impl ExpandExpression for ShorthandBlock { #[derive(Debug, Copy, Clone)] pub struct ShorthandPath; +impl FallibleColorSyntax for ShorthandPath { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + token_nodes.atomic(|token_nodes| { + let variable = color_fallible_syntax(&VariablePathShape, token_nodes, context, shapes); + + match variable { + Ok(_) => { + // if it's a variable path, that's the head part + return Ok(()); + } + + Err(_) => { + // otherwise, we'll try to find a member path + } + } + + // look for a member (`` -> `$it.`) + color_fallible_syntax(&MemberShape, token_nodes, context, shapes)?; + + // Now that we've synthesized the head, of the path, proceed to expand the tail of the path + // like any other path. + let tail = color_fallible_syntax(&PathTailShape, token_nodes, context, shapes); + + match tail { + Ok(_) => {} + Err(_) => { + // It's ok if there's no path tail; a single member is sufficient + } + } + + Ok(()) + }) + } +} + impl ExpandExpression for ShorthandPath { fn expand_expr<'a, 'b>( &self, @@ -92,8 +213,6 @@ impl ExpandExpression for ShorthandPath { head = hir::Expression::dot_member(head, member); } - println!("{:?}", head); - Ok(head) } } @@ -104,6 +223,49 @@ impl ExpandExpression for ShorthandPath { #[derive(Debug, Copy, Clone)] pub struct ShorthandHeadShape; +impl FallibleColorSyntax for ShorthandHeadShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + // A shorthand path must not be at EOF + let peeked = token_nodes.peek_non_ws().not_eof("shorthand path")?; + + match peeked.node { + // If the head of a shorthand path is a bare token, it expands to `$it.bare` + TokenNode::Token(Tagged { + item: RawToken::Bare, + tag, + }) => { + peeked.commit(); + shapes.push(FlatShape::BareMember.tagged(tag)); + Ok(()) + } + + // If the head of a shorthand path is a string, it expands to `$it."some string"` + TokenNode::Token(Tagged { + item: RawToken::String(_), + tag: outer, + }) => { + peeked.commit(); + shapes.push(FlatShape::StringMember.tagged(outer)); + Ok(()) + } + + other => Err(ShellError::type_error( + "shorthand head", + other.tagged_type_name(), + )), + } + } +} + impl ExpandExpression for ShorthandHeadShape { fn expand_expr<'a, 'b>( &self, diff --git a/src/parser/hir/syntax_shape/expression.rs b/src/parser/hir/syntax_shape/expression.rs index 58cfa4a1a5..fc99c38dc3 100644 --- a/src/parser/hir/syntax_shape/expression.rs +++ b/src/parser/hir/syntax_shape/expression.rs @@ -1,3 +1,4 @@ +pub(crate) mod atom; pub(crate) mod delimited; pub(crate) mod file_path; pub(crate) mod list; @@ -8,14 +9,14 @@ pub(crate) mod unit; pub(crate) mod variable_path; use crate::parser::hir::syntax_shape::{ - expand_expr, expand_syntax, expand_variable, expression::delimited::expand_delimited_expr, - BareShape, DotShape, ExpandContext, ExpandExpression, ExpandSyntax, ExpressionContinuation, - ExpressionContinuationShape, UnitShape, + color_delimited_square, color_fallible_syntax, color_fallible_syntax_with, expand_atom, + expand_delimited_square, expand_expr, expand_syntax, AtomicToken, BareShape, ColorableDotShape, + DotShape, ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, ExpressionContinuation, + ExpressionContinuationShape, FallibleColorSyntax, FlatShape, }; use crate::parser::{ hir, - hir::{Expression, Operator, TokensIterator}, - RawToken, Token, TokenNode, + hir::{Expression, TokensIterator}, }; use crate::prelude::*; use std::path::PathBuf; @@ -36,6 +37,32 @@ impl ExpandExpression for AnyExpressionShape { } } +impl FallibleColorSyntax for AnyExpressionShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + // Look for an expression at the cursor + color_fallible_syntax(&AnyExpressionStartShape, token_nodes, context, shapes)?; + + match continue_coloring_expression(token_nodes, context, shapes) { + Err(_) => { + // it's fine for there to be no continuation + } + + Ok(()) => {} + } + + Ok(()) + } +} + pub(crate) fn continue_expression( mut head: hir::Expression, token_nodes: &mut TokensIterator<'_>, @@ -64,6 +91,30 @@ pub(crate) fn continue_expression( } } +pub(crate) fn continue_coloring_expression( + token_nodes: &mut TokensIterator<'_>, + context: &ExpandContext, + shapes: &mut Vec>, +) -> Result<(), ShellError> { + // if there's not even one expression continuation, fail + color_fallible_syntax(&ExpressionContinuationShape, token_nodes, context, shapes)?; + + loop { + // Check to see whether there's any continuation after the head expression + let result = + color_fallible_syntax(&ExpressionContinuationShape, token_nodes, context, shapes); + + match result { + Err(_) => { + // We already saw one continuation, so just return + return Ok(()); + } + + Ok(_) => {} + } + } +} + #[derive(Debug, Copy, Clone)] pub struct AnyExpressionStartShape; @@ -73,59 +124,148 @@ impl ExpandExpression for AnyExpressionStartShape { token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, ) -> Result { - let size = expand_expr(&UnitShape, token_nodes, context); + let atom = expand_atom(token_nodes, "expression", context, ExpansionRule::new())?; - match size { - Ok(expr) => return Ok(expr), - Err(_) => {} - } - - let peek_next = token_nodes.peek_any().not_eof("expression")?; - - let head = match peek_next.node { - TokenNode::Token(token) => match token.item { - RawToken::Bare | RawToken::Operator(Operator::Dot) => { - let start = token.tag; - peek_next.commit(); - - let end = expand_syntax(&BareTailShape, token_nodes, context)?; - - match end { - Some(end) => return Ok(hir::Expression::bare(start.until(end))), - None => return Ok(hir::Expression::bare(start)), - } - } - _ => { - peek_next.commit(); - expand_one_context_free_token(*token, context) - } - }, - node @ TokenNode::Call(_) - | node @ TokenNode::Nodes(_) - | node @ TokenNode::Pipeline(_) - | node @ TokenNode::Flag(_) - | node @ TokenNode::Member(_) - | node @ TokenNode::Whitespace(_) => { - return Err(ShellError::type_error( - "expression", - node.tagged_type_name(), + match atom.item { + AtomicToken::Size { number, unit } => { + return Ok(hir::Expression::size( + number.to_number(context.source), + unit.item, + atom.tag, )) } - TokenNode::Delimited(delimited) => { - peek_next.commit(); - expand_delimited_expr(delimited, context) + + AtomicToken::SquareDelimited { nodes, .. } => { + expand_delimited_square(&nodes, atom.tag, context) } - TokenNode::Error(error) => return Err(*error.item.clone()), - }?; + AtomicToken::Word { .. } | AtomicToken::Dot { .. } => { + let end = expand_syntax(&BareTailShape, token_nodes, context)?; + Ok(hir::Expression::bare(atom.tag.until_option(end))) + } - Ok(head) + other => return other.tagged(atom.tag).into_hir(context, "expression"), + } + } +} + +impl FallibleColorSyntax for AnyExpressionStartShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let atom = token_nodes.spanned(|token_nodes| { + expand_atom( + token_nodes, + "expression", + context, + ExpansionRule::permissive(), + ) + }); + + let atom = match atom { + Tagged { + item: Err(_err), + tag, + } => { + shapes.push(FlatShape::Error.tagged(tag)); + return Ok(()); + } + + Tagged { + item: Ok(value), .. + } => value, + }; + + match atom.item { + AtomicToken::Size { number, unit } => shapes.push( + FlatShape::Size { + number: number.tag, + unit: unit.tag, + } + .tagged(atom.tag), + ), + + AtomicToken::SquareDelimited { nodes, tags } => { + color_delimited_square(tags, &nodes, atom.tag, context, shapes) + } + + AtomicToken::Word { .. } | AtomicToken::Dot { .. } => { + shapes.push(FlatShape::Word.tagged(atom.tag)); + } + + _ => atom.color_tokens(shapes), + } + + Ok(()) } } #[derive(Debug, Copy, Clone)] pub struct BareTailShape; +impl FallibleColorSyntax for BareTailShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let len = shapes.len(); + + loop { + let word = color_fallible_syntax_with( + &BareShape, + &FlatShape::Word, + token_nodes, + context, + shapes, + ); + + match word { + // if a word was found, continue + Ok(_) => continue, + // if a word wasn't found, try to find a dot + Err(_) => {} + } + + // try to find a dot + let dot = color_fallible_syntax_with( + &ColorableDotShape, + &FlatShape::Word, + token_nodes, + context, + shapes, + ); + + match dot { + // if a dot was found, try to find another word + Ok(_) => continue, + // otherwise, we're done + Err(_) => break, + } + } + + if shapes.len() > len { + Ok(()) + } else { + Err(ShellError::syntax_error( + "No tokens matched BareTailShape".tagged_unknown(), + )) + } + } +} + impl ExpandSyntax for BareTailShape { type Output = Option; @@ -158,29 +298,6 @@ impl ExpandSyntax for BareTailShape { } } -fn expand_one_context_free_token<'a, 'b>( - token: Token, - context: &ExpandContext, -) -> Result { - Ok(match token.item { - RawToken::Number(number) => { - hir::Expression::number(number.to_number(context.source), token.tag) - } - RawToken::Operator(..) => { - return Err(ShellError::syntax_error( - "unexpected operator, expected an expression".tagged(token.tag), - )) - } - RawToken::Size(..) => unimplemented!("size"), - RawToken::String(tag) => hir::Expression::string(tag, token.tag), - RawToken::Variable(tag) => expand_variable(tag, token.tag, &context.source), - RawToken::ExternalCommand(_) => unimplemented!(), - RawToken::ExternalWord => unimplemented!(), - RawToken::GlobPattern => hir::Expression::pattern(token.tag), - RawToken::Bare => hir::Expression::string(token.tag, token.tag), - }) -} - pub fn expand_file_path(string: &str, context: &ExpandContext) -> PathBuf { let expanded = shellexpand::tilde_with_context(string, || context.homedir()); diff --git a/src/parser/hir/syntax_shape/expression/atom.rs b/src/parser/hir/syntax_shape/expression/atom.rs new file mode 100644 index 0000000000..83306da741 --- /dev/null +++ b/src/parser/hir/syntax_shape/expression/atom.rs @@ -0,0 +1,541 @@ +use crate::parser::hir::syntax_shape::{ + expand_syntax, expression::expand_file_path, parse_single_node, BarePathShape, + BarePatternShape, ExpandContext, UnitShape, +}; +use crate::parser::{ + hir, + hir::{Expression, RawNumber, TokensIterator}, + parse::flag::{Flag, FlagKind}, + DelimitedNode, Delimiter, FlatShape, RawToken, TokenNode, Unit, +}; +use crate::prelude::*; + +#[derive(Debug)] +pub enum AtomicToken<'tokens> { + Eof { + tag: Tag, + }, + Error { + error: Tagged, + }, + Number { + number: RawNumber, + }, + Size { + number: Tagged, + unit: Tagged, + }, + String { + body: Tag, + }, + ItVariable { + name: Tag, + }, + Variable { + name: Tag, + }, + ExternalCommand { + command: Tag, + }, + ExternalWord { + text: Tag, + }, + GlobPattern { + pattern: Tag, + }, + FilePath { + path: Tag, + }, + Word { + text: Tag, + }, + SquareDelimited { + tags: (Tag, Tag), + nodes: &'tokens Vec, + }, + ParenDelimited { + tags: (Tag, Tag), + nodes: &'tokens Vec, + }, + BraceDelimited { + tags: (Tag, Tag), + nodes: &'tokens Vec, + }, + Pipeline { + pipe: Option, + elements: Tagged<&'tokens Vec>, + }, + ShorthandFlag { + name: Tag, + }, + LonghandFlag { + name: Tag, + }, + Dot { + text: Tag, + }, + Operator { + text: Tag, + }, + Whitespace { + text: Tag, + }, +} + +pub type TaggedAtomicToken<'tokens> = Tagged>; + +impl<'tokens> TaggedAtomicToken<'tokens> { + pub fn into_hir( + &self, + context: &ExpandContext, + expected: &'static str, + ) -> Result { + Ok(match &self.item { + AtomicToken::Eof { .. } => { + return Err(ShellError::type_error( + expected, + "eof atomic token".tagged(self.tag), + )) + } + AtomicToken::Error { .. } => { + return Err(ShellError::type_error( + expected, + "eof atomic token".tagged(self.tag), + )) + } + AtomicToken::Operator { .. } => { + return Err(ShellError::type_error( + expected, + "operator".tagged(self.tag), + )) + } + AtomicToken::ShorthandFlag { .. } => { + return Err(ShellError::type_error( + expected, + "shorthand flag".tagged(self.tag), + )) + } + AtomicToken::LonghandFlag { .. } => { + return Err(ShellError::type_error(expected, "flag".tagged(self.tag))) + } + AtomicToken::Whitespace { .. } => { + return Err(ShellError::unimplemented("whitespace in AtomicToken")) + } + AtomicToken::Dot { .. } => { + return Err(ShellError::type_error(expected, "dot".tagged(self.tag))) + } + AtomicToken::Number { number } => { + Expression::number(number.to_number(context.source), self.tag) + } + AtomicToken::FilePath { path } => Expression::file_path( + expand_file_path(path.slice(context.source), context), + self.tag, + ), + AtomicToken::Size { number, unit } => { + Expression::size(number.to_number(context.source), **unit, self.tag) + } + AtomicToken::String { body } => Expression::string(body, self.tag), + AtomicToken::ItVariable { name } => Expression::it_variable(name, self.tag), + AtomicToken::Variable { name } => Expression::variable(name, self.tag), + AtomicToken::ExternalCommand { command } => { + Expression::external_command(command, self.tag) + } + AtomicToken::ExternalWord { text } => Expression::string(text, self.tag), + AtomicToken::GlobPattern { pattern } => Expression::pattern(pattern), + AtomicToken::Word { text } => Expression::string(text, text), + AtomicToken::SquareDelimited { .. } => unimplemented!("into_hir"), + AtomicToken::ParenDelimited { .. } => unimplemented!("into_hir"), + AtomicToken::BraceDelimited { .. } => unimplemented!("into_hir"), + AtomicToken::Pipeline { .. } => unimplemented!("into_hir"), + }) + } + + pub fn tagged_type_name(&self) -> Tagged<&'static str> { + match &self.item { + AtomicToken::Eof { .. } => "eof", + AtomicToken::Error { .. } => "error", + AtomicToken::Operator { .. } => "operator", + AtomicToken::ShorthandFlag { .. } => "shorthand flag", + AtomicToken::LonghandFlag { .. } => "flag", + AtomicToken::Whitespace { .. } => "whitespace", + AtomicToken::Dot { .. } => "dot", + AtomicToken::Number { .. } => "number", + AtomicToken::FilePath { .. } => "file path", + AtomicToken::Size { .. } => "size", + AtomicToken::String { .. } => "string", + AtomicToken::ItVariable { .. } => "$it", + AtomicToken::Variable { .. } => "variable", + AtomicToken::ExternalCommand { .. } => "external command", + AtomicToken::ExternalWord { .. } => "external word", + AtomicToken::GlobPattern { .. } => "file pattern", + AtomicToken::Word { .. } => "word", + AtomicToken::SquareDelimited { .. } => "array literal", + AtomicToken::ParenDelimited { .. } => "parenthesized expression", + AtomicToken::BraceDelimited { .. } => "block", + AtomicToken::Pipeline { .. } => "pipeline", + } + .tagged(self.tag) + } + + pub(crate) fn color_tokens(&self, shapes: &mut Vec>) { + match &self.item { + AtomicToken::Eof { .. } => {} + AtomicToken::Error { .. } => return shapes.push(FlatShape::Error.tagged(self.tag)), + AtomicToken::Operator { .. } => { + return shapes.push(FlatShape::Operator.tagged(self.tag)); + } + AtomicToken::ShorthandFlag { .. } => { + return shapes.push(FlatShape::ShorthandFlag.tagged(self.tag)); + } + AtomicToken::LonghandFlag { .. } => { + return shapes.push(FlatShape::Flag.tagged(self.tag)); + } + AtomicToken::Whitespace { .. } => { + return shapes.push(FlatShape::Whitespace.tagged(self.tag)); + } + AtomicToken::FilePath { .. } => return shapes.push(FlatShape::Path.tagged(self.tag)), + AtomicToken::Dot { .. } => return shapes.push(FlatShape::Dot.tagged(self.tag)), + AtomicToken::Number { + number: RawNumber::Decimal(_), + } => { + return shapes.push(FlatShape::Decimal.tagged(self.tag)); + } + AtomicToken::Number { + number: RawNumber::Int(_), + } => { + return shapes.push(FlatShape::Int.tagged(self.tag)); + } + AtomicToken::Size { number, unit } => { + return shapes.push( + FlatShape::Size { + number: number.tag, + unit: unit.tag, + } + .tagged(self.tag), + ); + } + AtomicToken::String { .. } => return shapes.push(FlatShape::String.tagged(self.tag)), + AtomicToken::ItVariable { .. } => { + return shapes.push(FlatShape::ItVariable.tagged(self.tag)) + } + AtomicToken::Variable { .. } => { + return shapes.push(FlatShape::Variable.tagged(self.tag)) + } + AtomicToken::ExternalCommand { .. } => { + return shapes.push(FlatShape::ExternalCommand.tagged(self.tag)); + } + AtomicToken::ExternalWord { .. } => { + return shapes.push(FlatShape::ExternalWord.tagged(self.tag)) + } + AtomicToken::GlobPattern { .. } => { + return shapes.push(FlatShape::GlobPattern.tagged(self.tag)) + } + AtomicToken::Word { .. } => return shapes.push(FlatShape::Word.tagged(self.tag)), + _ => return shapes.push(FlatShape::Error.tagged(self.tag)), + } + } +} + +#[derive(Debug)] +pub enum WhitespaceHandling { + #[allow(unused)] + AllowWhitespace, + RejectWhitespace, +} + +#[derive(Debug)] +pub struct ExpansionRule { + pub(crate) allow_external_command: bool, + pub(crate) allow_external_word: bool, + pub(crate) allow_operator: bool, + pub(crate) allow_eof: bool, + pub(crate) treat_size_as_word: bool, + pub(crate) commit_errors: bool, + pub(crate) whitespace: WhitespaceHandling, +} + +impl ExpansionRule { + pub fn new() -> ExpansionRule { + ExpansionRule { + allow_external_command: false, + allow_external_word: false, + allow_operator: false, + allow_eof: false, + treat_size_as_word: false, + commit_errors: false, + whitespace: WhitespaceHandling::RejectWhitespace, + } + } + + /// The intent of permissive mode is to return an atomic token for every possible + /// input token. This is important for error-correcting parsing, such as the + /// syntax highlighter. + pub fn permissive() -> ExpansionRule { + ExpansionRule { + allow_external_command: true, + allow_external_word: true, + allow_operator: true, + allow_eof: true, + treat_size_as_word: false, + commit_errors: true, + whitespace: WhitespaceHandling::AllowWhitespace, + } + } + + #[allow(unused)] + pub fn allow_external_command(mut self) -> ExpansionRule { + self.allow_external_command = true; + self + } + + #[allow(unused)] + pub fn allow_operator(mut self) -> ExpansionRule { + self.allow_operator = true; + self + } + + #[allow(unused)] + pub fn no_operator(mut self) -> ExpansionRule { + self.allow_operator = false; + self + } + + #[allow(unused)] + pub fn no_external_command(mut self) -> ExpansionRule { + self.allow_external_command = false; + self + } + + #[allow(unused)] + pub fn allow_external_word(mut self) -> ExpansionRule { + self.allow_external_word = true; + self + } + + #[allow(unused)] + pub fn no_external_word(mut self) -> ExpansionRule { + self.allow_external_word = false; + self + } + + #[allow(unused)] + pub fn treat_size_as_word(mut self) -> ExpansionRule { + self.treat_size_as_word = true; + self + } + + #[allow(unused)] + pub fn commit_errors(mut self) -> ExpansionRule { + self.commit_errors = true; + self + } + + #[allow(unused)] + pub fn allow_whitespace(mut self) -> ExpansionRule { + self.whitespace = WhitespaceHandling::AllowWhitespace; + self + } + + #[allow(unused)] + pub fn reject_whitespace(mut self) -> ExpansionRule { + self.whitespace = WhitespaceHandling::RejectWhitespace; + self + } +} + +/// If the caller of expand_atom throws away the returned atomic token returned, it +/// must use a checkpoint to roll it back. +pub fn expand_atom<'me, 'content>( + token_nodes: &'me mut TokensIterator<'content>, + expected: &'static str, + context: &ExpandContext, + rule: ExpansionRule, +) -> Result, ShellError> { + if token_nodes.at_end() { + match rule.allow_eof { + true => { + return Ok(AtomicToken::Eof { + tag: Tag::unknown(), + } + .tagged_unknown()) + } + false => return Err(ShellError::unexpected_eof("anything", Tag::unknown())), + } + } + + // First, we'll need to handle the situation where more than one token corresponds + // to a single atomic token + + // If treat_size_as_word, don't try to parse the head of the token stream + // as a size. + match rule.treat_size_as_word { + true => {} + false => match expand_syntax(&UnitShape, token_nodes, context) { + // If the head of the stream isn't a valid unit, we'll try to parse + // it again next as a word + Err(_) => {} + + // But if it was a valid unit, we're done here + Ok(Tagged { + item: (number, unit), + tag, + }) => return Ok(AtomicToken::Size { number, unit }.tagged(tag)), + }, + } + + // Try to parse the head of the stream as a bare path. A bare path includes + // words as well as `.`s, connected together without whitespace. + match expand_syntax(&BarePathShape, token_nodes, context) { + // If we didn't find a bare path + Err(_) => {} + Ok(tag) => { + let next = token_nodes.peek_any(); + + match next.node { + Some(token) if token.is_pattern() => { + // if the very next token is a pattern, we're looking at a glob, not a + // word, and we should try to parse it as a glob next + } + + _ => return Ok(AtomicToken::Word { text: tag }.tagged(tag)), + } + } + } + + // Try to parse the head of the stream as a pattern. A pattern includes + // words, words with `*` as well as `.`s, connected together without whitespace. + match expand_syntax(&BarePatternShape, token_nodes, context) { + // If we didn't find a bare path + Err(_) => {} + Ok(tag) => return Ok(AtomicToken::GlobPattern { pattern: tag }.tagged(tag)), + } + + // The next token corresponds to at most one atomic token + + // We need to `peek` because `parse_single_node` doesn't cover all of the + // cases that `expand_atom` covers. We should probably collapse the two + // if possible. + let peeked = token_nodes.peek_any().not_eof(expected)?; + + match peeked.node { + TokenNode::Token(_) => { + // handle this next + } + + TokenNode::Error(error) => { + peeked.commit(); + return Ok(AtomicToken::Error { + error: error.clone(), + } + .tagged(error.tag)); + } + + // [ ... ] + TokenNode::Delimited(Tagged { + item: + DelimitedNode { + delimiter: Delimiter::Square, + tags, + children, + }, + tag, + }) => { + peeked.commit(); + return Ok(AtomicToken::SquareDelimited { + nodes: children, + tags: *tags, + } + .tagged(tag)); + } + + TokenNode::Flag(Tagged { + item: + Flag { + kind: FlagKind::Shorthand, + name, + }, + tag, + }) => { + peeked.commit(); + return Ok(AtomicToken::ShorthandFlag { name: *name }.tagged(tag)); + } + + TokenNode::Flag(Tagged { + item: + Flag { + kind: FlagKind::Longhand, + name, + }, + tag, + }) => { + peeked.commit(); + return Ok(AtomicToken::ShorthandFlag { name: *name }.tagged(tag)); + } + + // If we see whitespace, process the whitespace according to the whitespace + // handling rules + TokenNode::Whitespace(tag) => match rule.whitespace { + // if whitespace is allowed, return a whitespace token + WhitespaceHandling::AllowWhitespace => { + peeked.commit(); + return Ok(AtomicToken::Whitespace { text: *tag }.tagged(tag)); + } + + // if whitespace is disallowed, return an error + WhitespaceHandling::RejectWhitespace => { + return Err(ShellError::syntax_error( + "Unexpected whitespace".tagged(tag), + )) + } + }, + + other => { + let tag = peeked.node.tag(); + + peeked.commit(); + return Ok(AtomicToken::Error { + error: ShellError::type_error("token", other.tagged_type_name()).tagged(tag), + } + .tagged(tag)); + } + } + + parse_single_node(token_nodes, expected, |token, token_tag, err| { + Ok(match token { + // First, the error cases. Each error case corresponds to a expansion rule + // flag that can be used to allow the case + + // rule.allow_operator + RawToken::Operator(_) if !rule.allow_operator => return Err(err.error()), + // rule.allow_external_command + RawToken::ExternalCommand(_) if !rule.allow_external_command => { + return Err(ShellError::type_error( + expected, + token.type_name().tagged(token_tag), + )) + } + // rule.allow_external_word + RawToken::ExternalWord if !rule.allow_external_word => { + return Err(ShellError::invalid_external_word(token_tag)) + } + + RawToken::Number(number) => AtomicToken::Number { number }.tagged(token_tag), + RawToken::Operator(_) => AtomicToken::Operator { text: token_tag }.tagged(token_tag), + RawToken::String(body) => AtomicToken::String { body }.tagged(token_tag), + RawToken::Variable(name) if name.slice(context.source) == "it" => { + AtomicToken::ItVariable { name }.tagged(token_tag) + } + RawToken::Variable(name) => AtomicToken::Variable { name }.tagged(token_tag), + RawToken::ExternalCommand(command) => { + AtomicToken::ExternalCommand { command }.tagged(token_tag) + } + RawToken::ExternalWord => { + AtomicToken::ExternalWord { text: token_tag }.tagged(token_tag) + } + RawToken::GlobPattern => { + AtomicToken::GlobPattern { pattern: token_tag }.tagged(token_tag) + } + RawToken::Bare => AtomicToken::Word { text: token_tag }.tagged(token_tag), + }) + }) +} diff --git a/src/parser/hir/syntax_shape/expression/delimited.rs b/src/parser/hir/syntax_shape/expression/delimited.rs index 0a01b0fc26..001e3812f4 100644 --- a/src/parser/hir/syntax_shape/expression/delimited.rs +++ b/src/parser/hir/syntax_shape/expression/delimited.rs @@ -1,38 +1,49 @@ -use crate::parser::hir::syntax_shape::{expand_syntax, ExpandContext, ExpressionListShape}; -use crate::parser::{hir, hir::TokensIterator}; -use crate::parser::{DelimitedNode, Delimiter}; +use crate::parser::hir::syntax_shape::{ + color_syntax, expand_syntax, ColorSyntax, ExpandContext, ExpressionListShape, TokenNode, +}; +use crate::parser::{hir, hir::TokensIterator, Delimiter, FlatShape}; use crate::prelude::*; -pub fn expand_delimited_expr( - delimited: &Tagged, +pub fn expand_delimited_square( + children: &Vec, + tag: Tag, context: &ExpandContext, ) -> Result { - match &delimited.item { - DelimitedNode { - delimiter: Delimiter::Square, - children, - } => { - let mut tokens = TokensIterator::new(&children, delimited.tag, false); + let mut tokens = TokensIterator::new(&children, tag, false); - let list = expand_syntax(&ExpressionListShape, &mut tokens, context); + let list = expand_syntax(&ExpressionListShape, &mut tokens, context); - Ok(hir::Expression::list(list?, delimited.tag)) - } + Ok(hir::Expression::list(list?, tag)) +} - DelimitedNode { - delimiter: Delimiter::Paren, - .. - } => Err(ShellError::type_error( - "expression", - "unimplemented call expression".tagged(delimited.tag), - )), +pub fn color_delimited_square( + (open, close): (Tag, Tag), + children: &Vec, + tag: Tag, + context: &ExpandContext, + shapes: &mut Vec>, +) { + shapes.push(FlatShape::OpenDelimiter(Delimiter::Square).tagged(open)); + let mut tokens = TokensIterator::new(&children, tag, false); + let _list = color_syntax(&ExpressionListShape, &mut tokens, context, shapes); + shapes.push(FlatShape::CloseDelimiter(Delimiter::Square).tagged(close)); +} - DelimitedNode { - delimiter: Delimiter::Brace, - .. - } => Err(ShellError::type_error( - "expression", - "unimplemented block expression".tagged(delimited.tag), - )), +#[derive(Debug, Copy, Clone)] +pub struct DelimitedShape; + +impl ColorSyntax for DelimitedShape { + type Info = (); + type Input = (Delimiter, Tag, Tag); + fn color_syntax<'a, 'b>( + &self, + (delimiter, open, close): &(Delimiter, Tag, Tag), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Self::Info { + shapes.push(FlatShape::OpenDelimiter(*delimiter).tagged(open)); + color_syntax(&ExpressionListShape, token_nodes, context, shapes); + shapes.push(FlatShape::CloseDelimiter(*delimiter).tagged(close)); } } diff --git a/src/parser/hir/syntax_shape/expression/file_path.rs b/src/parser/hir/syntax_shape/expression/file_path.rs index c0e5c7c2ab..e73dc8d647 100644 --- a/src/parser/hir/syntax_shape/expression/file_path.rs +++ b/src/parser/hir/syntax_shape/expression/file_path.rs @@ -1,59 +1,71 @@ +use crate::parser::hir::syntax_shape::expression::atom::{expand_atom, AtomicToken, ExpansionRule}; use crate::parser::hir::syntax_shape::{ - expand_syntax, expression::expand_file_path, parse_single_node, BarePathShape, ExpandContext, - ExpandExpression, + expression::expand_file_path, ExpandContext, ExpandExpression, FallibleColorSyntax, FlatShape, }; -use crate::parser::{hir, hir::TokensIterator, RawToken}; +use crate::parser::{hir, hir::TokensIterator}; use crate::prelude::*; #[derive(Debug, Copy, Clone)] pub struct FilePathShape; +impl FallibleColorSyntax for FilePathShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let atom = expand_atom( + token_nodes, + "file path", + context, + ExpansionRule::permissive(), + ); + + let atom = match atom { + Err(_) => return Ok(()), + Ok(atom) => atom, + }; + + match atom.item { + AtomicToken::Word { .. } + | AtomicToken::String { .. } + | AtomicToken::Number { .. } + | AtomicToken::Size { .. } => { + shapes.push(FlatShape::Path.tagged(atom.tag)); + } + + _ => atom.color_tokens(shapes), + } + + Ok(()) + } +} + impl ExpandExpression for FilePathShape { fn expand_expr<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, ) -> Result { - let bare = expand_syntax(&BarePathShape, token_nodes, context); + let atom = expand_atom(token_nodes, "file path", context, ExpansionRule::new())?; - match bare { - Ok(tag) => { - let string = tag.slice(context.source); - let path = expand_file_path(string, context); - return Ok(hir::Expression::file_path(path, tag)); + match atom.item { + AtomicToken::Word { text: body } | AtomicToken::String { body } => { + let path = expand_file_path(body.slice(context.source), context); + return Ok(hir::Expression::file_path(path, atom.tag)); } - Err(_) => {} + + AtomicToken::Number { .. } | AtomicToken::Size { .. } => { + let path = atom.tag.slice(context.source); + return Ok(hir::Expression::file_path(path, atom.tag)); + } + + _ => return atom.into_hir(context, "file path"), } - - parse_single_node(token_nodes, "Path", |token, token_tag| { - Ok(match token { - RawToken::GlobPattern => { - return Err(ShellError::type_error( - "Path", - "glob pattern".tagged(token_tag), - )) - } - RawToken::Operator(..) => { - return Err(ShellError::type_error("Path", "operator".tagged(token_tag))) - } - RawToken::Variable(tag) if tag.slice(context.source) == "it" => { - hir::Expression::it_variable(tag, token_tag) - } - RawToken::Variable(tag) => hir::Expression::variable(tag, token_tag), - RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token_tag), - RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token_tag)), - RawToken::Number(_) => hir::Expression::bare(token_tag), - RawToken::Size(_, _) => hir::Expression::bare(token_tag), - RawToken::Bare => hir::Expression::file_path( - expand_file_path(token_tag.slice(context.source), context), - token_tag, - ), - - RawToken::String(tag) => hir::Expression::file_path( - expand_file_path(tag.slice(context.source), context), - token_tag, - ), - }) - }) } } diff --git a/src/parser/hir/syntax_shape/expression/list.rs b/src/parser/hir/syntax_shape/expression/list.rs index 9d28f44141..4109108a37 100644 --- a/src/parser/hir/syntax_shape/expression/list.rs +++ b/src/parser/hir/syntax_shape/expression/list.rs @@ -2,10 +2,14 @@ use crate::errors::ShellError; use crate::parser::{ hir, hir::syntax_shape::{ - expand_expr, maybe_spaced, spaced, AnyExpressionShape, ExpandContext, ExpandSyntax, + color_fallible_syntax, color_syntax, expand_atom, expand_expr, maybe_spaced, spaced, + AnyExpressionShape, ColorSyntax, ExpandContext, ExpandSyntax, ExpansionRule, + MaybeSpaceShape, SpaceShape, }, - hir::{debug_tokens, TokensIterator}, + hir::TokensIterator, + FlatShape, }; +use crate::Tagged; #[derive(Debug, Copy, Clone)] pub struct ExpressionListShape; @@ -28,8 +32,6 @@ impl ExpandSyntax for ExpressionListShape { exprs.push(expr); - println!("{:?}", debug_tokens(token_nodes, context.source)); - loop { if token_nodes.at_end_possible_ws() { return Ok(exprs); @@ -41,3 +43,134 @@ impl ExpandSyntax for ExpressionListShape { } } } + +impl ColorSyntax for ExpressionListShape { + type Info = (); + type Input = (); + + /// The intent of this method is to fully color an expression list shape infallibly. + /// This means that if we can't expand a token into an expression, we fall back to + /// a simpler coloring strategy. + /// + /// This would apply to something like `where x >`, which includes an incomplete + /// binary operator. Since we will fail to process it as a binary operator, we'll + /// fall back to a simpler coloring and move on. + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) { + // We encountered a parsing error and will continue with simpler coloring ("backoff + // coloring mode") + let mut backoff = false; + + // Consume any leading whitespace + color_syntax(&MaybeSpaceShape, token_nodes, context, shapes); + + loop { + // If we reached the very end of the token stream, we're done + if token_nodes.at_end() { + return; + } + + if backoff { + let len = shapes.len(); + + // If we previously encountered a parsing error, use backoff coloring mode + color_syntax(&SimplestExpression, token_nodes, context, shapes); + + if len == shapes.len() && !token_nodes.at_end() { + // This should never happen, but if it does, a panic is better than an infinite loop + panic!("Unexpected tokens left that couldn't be colored even with SimplestExpression") + } + } else { + // Try to color the head of the stream as an expression + match color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes) { + // If no expression was found, switch to backoff coloring mode + Err(_) => { + backoff = true; + continue; + } + Ok(_) => {} + } + + // If an expression was found, consume a space + match color_fallible_syntax(&SpaceShape, token_nodes, context, shapes) { + Err(_) => { + // If no space was found, we're either at the end or there's an error. + // Either way, switch to backoff coloring mode. If we're at the end + // it won't have any consequences. + backoff = true; + } + Ok(_) => { + // Otherwise, move on to the next expression + } + } + } + } + } +} + +/// BackoffColoringMode consumes all of the remaining tokens in an infallible way +#[derive(Debug, Copy, Clone)] +pub struct BackoffColoringMode; + +impl ColorSyntax for BackoffColoringMode { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &Self::Input, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Self::Info { + loop { + if token_nodes.at_end() { + break; + } + + let len = shapes.len(); + color_syntax(&SimplestExpression, token_nodes, context, shapes); + + if len == shapes.len() && !token_nodes.at_end() { + // This shouldn't happen, but if it does, a panic is better than an infinite loop + panic!("SimplestExpression failed to consume any tokens, but it's not at the end. This is unexpected\n== token nodes==\n{:#?}\n\n== shapes ==\n{:#?}", token_nodes, shapes); + } + } + } +} + +/// The point of `SimplestExpression` is to serve as an infallible base case for coloring. +/// As a last ditch effort, if we can't find any way to parse the head of the stream as an +/// expression, fall back to simple coloring. +#[derive(Debug, Copy, Clone)] +pub struct SimplestExpression; + +impl ColorSyntax for SimplestExpression { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) { + let atom = expand_atom( + token_nodes, + "any token", + context, + ExpansionRule::permissive(), + ); + + match atom { + Err(_) => {} + Ok(atom) => atom.color_tokens(shapes), + } + } +} diff --git a/src/parser/hir/syntax_shape/expression/number.rs b/src/parser/hir/syntax_shape/expression/number.rs index 5b77044a2d..8d3cb048c6 100644 --- a/src/parser/hir/syntax_shape/expression/number.rs +++ b/src/parser/hir/syntax_shape/expression/number.rs @@ -1,4 +1,7 @@ -use crate::parser::hir::syntax_shape::{parse_single_node, ExpandContext, ExpandExpression}; +use crate::parser::hir::syntax_shape::{ + expand_atom, parse_single_node, ExpandContext, ExpandExpression, ExpansionRule, + FallibleColorSyntax, FlatShape, +}; use crate::parser::{ hir, hir::{RawNumber, TokensIterator}, @@ -15,20 +18,9 @@ impl ExpandExpression for NumberShape { token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, ) -> Result { - parse_single_node(token_nodes, "Number", |token, token_tag| { + parse_single_node(token_nodes, "Number", |token, token_tag, err| { Ok(match token { - RawToken::GlobPattern => { - return Err(ShellError::type_error( - "Number", - "glob pattern".to_string().tagged(token_tag), - )) - } - RawToken::Operator(..) => { - return Err(ShellError::type_error( - "Number", - "operator".to_string().tagged(token_tag), - )) - } + RawToken::GlobPattern | RawToken::Operator(..) => return Err(err.error()), RawToken::Variable(tag) if tag.slice(context.source) == "it" => { hir::Expression::it_variable(tag, token_tag) } @@ -38,9 +30,6 @@ impl ExpandExpression for NumberShape { RawToken::Number(number) => { hir::Expression::number(number.to_number(context.source), token_tag) } - RawToken::Size(number, unit) => { - hir::Expression::size(number.to_number(context.source), unit, token_tag) - } RawToken::Bare => hir::Expression::bare(token_tag), RawToken::String(tag) => hir::Expression::string(tag, token_tag), }) @@ -48,6 +37,35 @@ impl ExpandExpression for NumberShape { } } +impl FallibleColorSyntax for NumberShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let atom = token_nodes.spanned(|token_nodes| { + expand_atom(token_nodes, "number", context, ExpansionRule::permissive()) + }); + + let atom = match atom { + Tagged { item: Err(_), tag } => { + shapes.push(FlatShape::Error.tagged(tag)); + return Ok(()); + } + Tagged { item: Ok(atom), .. } => atom, + }; + + atom.color_tokens(shapes); + + Ok(()) + } +} + #[derive(Debug, Copy, Clone)] pub struct IntShape; @@ -57,41 +75,51 @@ impl ExpandExpression for IntShape { token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, ) -> Result { - parse_single_node(token_nodes, "Integer", |token, token_tag| { + parse_single_node(token_nodes, "Integer", |token, token_tag, err| { Ok(match token { - RawToken::GlobPattern => { - return Err(ShellError::type_error( - "Integer", - "glob pattern".to_string().tagged(token_tag), - )) - } - RawToken::Operator(..) => { - return Err(ShellError::type_error( - "Integer", - "operator".to_string().tagged(token_tag), - )) - } + RawToken::GlobPattern | RawToken::Operator(..) => return Err(err.error()), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token_tag)), RawToken::Variable(tag) if tag.slice(context.source) == "it" => { hir::Expression::it_variable(tag, token_tag) } RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token_tag), - RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token_tag)), RawToken::Variable(tag) => hir::Expression::variable(tag, token_tag), RawToken::Number(number @ RawNumber::Int(_)) => { hir::Expression::number(number.to_number(context.source), token_tag) } - token @ RawToken::Number(_) => { - return Err(ShellError::type_error( - "Integer", - token.type_name().tagged(token_tag), - )); - } - RawToken::Size(number, unit) => { - hir::Expression::size(number.to_number(context.source), unit, token_tag) - } + RawToken::Number(_) => return Err(err.error()), RawToken::Bare => hir::Expression::bare(token_tag), RawToken::String(tag) => hir::Expression::string(tag, token_tag), }) }) } } + +impl FallibleColorSyntax for IntShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let atom = token_nodes.spanned(|token_nodes| { + expand_atom(token_nodes, "integer", context, ExpansionRule::permissive()) + }); + + let atom = match atom { + Tagged { item: Err(_), tag } => { + shapes.push(FlatShape::Error.tagged(tag)); + return Ok(()); + } + Tagged { item: Ok(atom), .. } => atom, + }; + + atom.color_tokens(shapes); + + Ok(()) + } +} diff --git a/src/parser/hir/syntax_shape/expression/pattern.rs b/src/parser/hir/syntax_shape/expression/pattern.rs index 4105b79b4f..5c863de728 100644 --- a/src/parser/hir/syntax_shape/expression/pattern.rs +++ b/src/parser/hir/syntax_shape/expression/pattern.rs @@ -1,6 +1,7 @@ use crate::parser::hir::syntax_shape::{ - expand_bare, expand_syntax, expression::expand_file_path, parse_single_node, ExpandContext, - ExpandExpression, ExpandSyntax, + expand_atom, expand_bare, expand_syntax, expression::expand_file_path, parse_single_node, + AtomicToken, ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, + FlatShape, }; use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode}; use crate::prelude::*; @@ -8,6 +9,32 @@ use crate::prelude::*; #[derive(Debug, Copy, Clone)] pub struct PatternShape; +impl FallibleColorSyntax for PatternShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + token_nodes.atomic(|token_nodes| { + let atom = expand_atom(token_nodes, "pattern", context, ExpansionRule::permissive())?; + + match &atom.item { + AtomicToken::GlobPattern { .. } | AtomicToken::Word { .. } => { + shapes.push(FlatShape::GlobPattern.tagged(atom.tag)); + Ok(()) + } + + _ => Err(ShellError::type_error("pattern", atom.tagged_type_name())), + } + }) + } +} + impl ExpandExpression for PatternShape { fn expand_expr<'a, 'b>( &self, @@ -23,7 +50,7 @@ impl ExpandExpression for PatternShape { Err(_) => {} } - parse_single_node(token_nodes, "Pattern", |token, token_tag| { + parse_single_node(token_nodes, "Pattern", |token, token_tag, _| { Ok(match token { RawToken::GlobPattern => { return Err(ShellError::unreachable( @@ -44,7 +71,6 @@ impl ExpandExpression for PatternShape { RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token_tag), RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token_tag)), RawToken::Number(_) => hir::Expression::bare(token_tag), - RawToken::Size(_, _) => hir::Expression::bare(token_tag), RawToken::String(tag) => hir::Expression::file_path( expand_file_path(tag.slice(context.source), context), diff --git a/src/parser/hir/syntax_shape/expression/string.rs b/src/parser/hir/syntax_shape/expression/string.rs index 6a4973febe..6f33ae5eb1 100644 --- a/src/parser/hir/syntax_shape/expression/string.rs +++ b/src/parser/hir/syntax_shape/expression/string.rs @@ -1,5 +1,6 @@ use crate::parser::hir::syntax_shape::{ - expand_variable, parse_single_node, ExpandContext, ExpandExpression, TestSyntax, + expand_atom, expand_variable, parse_single_node, AtomicToken, ExpandContext, ExpandExpression, + ExpansionRule, FallibleColorSyntax, FlatShape, TestSyntax, }; use crate::parser::hir::tokens_iterator::Peeked; use crate::parser::{hir, hir::TokensIterator, RawToken, TokenNode}; @@ -8,13 +9,43 @@ use crate::prelude::*; #[derive(Debug, Copy, Clone)] pub struct StringShape; +impl FallibleColorSyntax for StringShape { + type Info = (); + type Input = FlatShape; + + fn color_syntax<'a, 'b>( + &self, + input: &FlatShape, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let atom = expand_atom(token_nodes, "string", context, ExpansionRule::permissive()); + + let atom = match atom { + Err(_) => return Ok(()), + Ok(atom) => atom, + }; + + match atom { + Tagged { + item: AtomicToken::String { .. }, + tag, + } => shapes.push((*input).tagged(tag)), + other => other.color_tokens(shapes), + } + + Ok(()) + } +} + impl ExpandExpression for StringShape { fn expand_expr<'a, 'b>( &self, token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, ) -> Result { - parse_single_node(token_nodes, "String", |token, token_tag| { + parse_single_node(token_nodes, "String", |token, token_tag, _| { Ok(match token { RawToken::GlobPattern => { return Err(ShellError::type_error( @@ -32,7 +63,6 @@ impl ExpandExpression for StringShape { RawToken::ExternalCommand(tag) => hir::Expression::external_command(tag, token_tag), RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token_tag)), RawToken::Number(_) => hir::Expression::bare(token_tag), - RawToken::Size(_, _) => hir::Expression::bare(token_tag), RawToken::Bare => hir::Expression::bare(token_tag), RawToken::String(tag) => hir::Expression::string(tag, token_tag), }) diff --git a/src/parser/hir/syntax_shape/expression/unit.rs b/src/parser/hir/syntax_shape/expression/unit.rs index cc3642bda5..65fca1a468 100644 --- a/src/parser/hir/syntax_shape/expression/unit.rs +++ b/src/parser/hir/syntax_shape/expression/unit.rs @@ -1,7 +1,8 @@ -use crate::parser::hir::syntax_shape::{ExpandContext, ExpandExpression}; +use crate::data::meta::Span; +use crate::parser::hir::syntax_shape::{ExpandContext, ExpandSyntax}; use crate::parser::parse::tokens::RawNumber; use crate::parser::parse::unit::Unit; -use crate::parser::{hir, hir::TokensIterator, RawToken, TokenNode}; +use crate::parser::{hir::TokensIterator, RawToken, TokenNode}; use crate::prelude::*; use nom::branch::alt; use nom::bytes::complete::tag; @@ -12,12 +13,14 @@ use nom::IResult; #[derive(Debug, Copy, Clone)] pub struct UnitShape; -impl ExpandExpression for UnitShape { - fn expand_expr<'a, 'b>( +impl ExpandSyntax for UnitShape { + type Output = Tagged<(Tagged, Tagged)>; + + fn expand_syntax<'a, 'b>( &self, token_nodes: &'b mut TokensIterator<'a>, context: &ExpandContext, - ) -> Result { + ) -> Result, Tagged)>, ShellError> { let peeked = token_nodes.peek_any().not_eof("unit")?; let tag = match peeked.node { @@ -40,15 +43,12 @@ impl ExpandExpression for UnitShape { Ok((number, unit)) => (number, unit), }; - Ok(hir::Expression::size( - number.to_number(context.source), - unit, - tag, - )) + peeked.commit(); + Ok((number, unit).tagged(tag)) } } -fn unit_size(input: &str, bare_tag: Tag) -> IResult<&str, (Tagged, Unit)> { +fn unit_size(input: &str, bare_tag: Tag) -> IResult<&str, (Tagged, Tagged)> { let (input, digits) = digit1(input)?; let (input, dot) = opt(tag("."))(input)?; @@ -85,5 +85,12 @@ fn unit_size(input: &str, bare_tag: Tag) -> IResult<&str, (Tagged, Un value(Unit::MB, alt((tag("PB"), tag("pb"), tag("Pb")))), )))(input)?; - Ok((input, (number, unit))) + let start_span = number.tag.span.end(); + + let unit_tag = Tag::new( + bare_tag.anchor, + Span::from((start_span, bare_tag.span.end())), + ); + + Ok((input, (number, unit.tagged(unit_tag)))) } diff --git a/src/parser/hir/syntax_shape/expression/variable_path.rs b/src/parser/hir/syntax_shape/expression/variable_path.rs index afea1b1499..a7f17a5971 100644 --- a/src/parser/hir/syntax_shape/expression/variable_path.rs +++ b/src/parser/hir/syntax_shape/expression/variable_path.rs @@ -1,6 +1,8 @@ use crate::parser::hir::syntax_shape::{ - expand_expr, expand_syntax, parse_single_node, AnyExpressionShape, BareShape, ExpandContext, - ExpandExpression, ExpandSyntax, Peeked, SkipSyntax, StringShape, TestSyntax, WhitespaceShape, + color_fallible_syntax, color_fallible_syntax_with, expand_atom, expand_expr, expand_syntax, + parse_single_node, AnyExpressionShape, AtomicToken, BareShape, ExpandContext, ExpandExpression, + ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, Peeked, SkipSyntax, StringShape, + TestSyntax, WhitespaceShape, }; use crate::parser::{hir, hir::Expression, hir::TokensIterator, Operator, RawToken}; use crate::prelude::*; @@ -42,9 +44,81 @@ impl ExpandExpression for VariablePathShape { } } +impl FallibleColorSyntax for VariablePathShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + token_nodes.atomic(|token_nodes| { + // If the head of the token stream is not a variable, fail + color_fallible_syntax(&VariableShape, token_nodes, context, shapes)?; + + loop { + // look for a dot at the head of a stream + let dot = color_fallible_syntax_with( + &ColorableDotShape, + &FlatShape::Dot, + token_nodes, + context, + shapes, + ); + + // if there's no dot, we're done + match dot { + Err(_) => break, + Ok(_) => {} + } + + // otherwise, look for a member, and if you don't find one, fail + color_fallible_syntax(&MemberShape, token_nodes, context, shapes)?; + } + + Ok(()) + }) + } +} + #[derive(Debug, Copy, Clone)] pub struct PathTailShape; +/// The failure mode of `PathTailShape` is a dot followed by a non-member +impl FallibleColorSyntax for PathTailShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + token_nodes.atomic(|token_nodes| loop { + let result = color_fallible_syntax_with( + &ColorableDotShape, + &FlatShape::Dot, + token_nodes, + context, + shapes, + ); + + match result { + Err(_) => return Ok(()), + Ok(_) => {} + } + + // If we've seen a dot but not a member, fail + color_fallible_syntax(&MemberShape, token_nodes, context, shapes)?; + }) + } +} + impl ExpandSyntax for PathTailShape { type Output = (Vec>, Tag); fn expand_syntax<'a, 'b>( @@ -121,6 +195,63 @@ impl ExpandSyntax for ExpressionContinuationShape { } } +pub enum ContinuationInfo { + Dot, + Infix, +} + +impl FallibleColorSyntax for ExpressionContinuationShape { + type Info = ContinuationInfo; + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result { + token_nodes.atomic(|token_nodes| { + // Try to expand a `.` + let dot = color_fallible_syntax_with( + &ColorableDotShape, + &FlatShape::Dot, + token_nodes, + context, + shapes, + ); + + match dot { + Ok(_) => { + // we found a dot, so let's keep looking for a member; if no member was found, fail + color_fallible_syntax(&MemberShape, token_nodes, context, shapes)?; + + Ok(ContinuationInfo::Dot) + } + Err(_) => { + let mut new_shapes = vec![]; + let result = token_nodes.atomic(|token_nodes| { + // we didn't find a dot, so let's see if we're looking at an infix. If not found, fail + color_fallible_syntax(&InfixShape, token_nodes, context, &mut new_shapes)?; + + // now that we've seen an infix shape, look for any expression. If not found, fail + color_fallible_syntax( + &AnyExpressionShape, + token_nodes, + context, + &mut new_shapes, + )?; + + Ok(ContinuationInfo::Infix) + })?; + shapes.extend(new_shapes); + Ok(result) + } + } + }) + } +} + #[derive(Debug, Copy, Clone)] pub struct VariableShape; @@ -130,7 +261,7 @@ impl ExpandExpression for VariableShape { token_nodes: &mut TokensIterator<'_>, context: &ExpandContext, ) -> Result { - parse_single_node(token_nodes, "variable", |token, token_tag| { + parse_single_node(token_nodes, "variable", |token, token_tag, _| { Ok(match token { RawToken::Variable(tag) => { if tag.slice(context.source) == "it" { @@ -150,6 +281,43 @@ impl ExpandExpression for VariableShape { } } +impl FallibleColorSyntax for VariableShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let atom = expand_atom( + token_nodes, + "variable", + context, + ExpansionRule::permissive(), + ); + + let atom = match atom { + Err(err) => return Err(err), + Ok(atom) => atom, + }; + + match &atom.item { + AtomicToken::Variable { .. } => { + shapes.push(FlatShape::Variable.tagged(atom.tag)); + Ok(()) + } + AtomicToken::ItVariable { .. } => { + shapes.push(FlatShape::ItVariable.tagged(atom.tag)); + Ok(()) + } + _ => Err(ShellError::type_error("variable", atom.tagged_type_name())), + } + } +} + #[derive(Debug, Clone, Copy)] pub enum Member { String(/* outer */ Tag, /* inner */ Tag), @@ -272,6 +440,55 @@ pub fn expand_column_path<'a, 'b>( #[derive(Debug, Copy, Clone)] pub struct ColumnPathShape; +impl FallibleColorSyntax for ColumnPathShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + // If there's not even one member shape, fail + color_fallible_syntax(&MemberShape, token_nodes, context, shapes)?; + + loop { + let checkpoint = token_nodes.checkpoint(); + + match color_fallible_syntax_with( + &ColorableDotShape, + &FlatShape::Dot, + checkpoint.iterator, + context, + shapes, + ) { + Err(_) => { + // we already saw at least one member shape, so return successfully + return Ok(()); + } + + Ok(_) => { + match color_fallible_syntax(&MemberShape, checkpoint.iterator, context, shapes) + { + Err(_) => { + // we saw a dot but not a member (but we saw at least one member), + // so don't commit the dot but return successfully + return Ok(()); + } + + Ok(_) => { + // we saw a dot and a member, so commit it and continue on + checkpoint.commit(); + } + } + } + } + } + } +} + impl ExpandSyntax for ColumnPathShape { type Output = Tagged>; @@ -287,6 +504,43 @@ impl ExpandSyntax for ColumnPathShape { #[derive(Debug, Copy, Clone)] pub struct MemberShape; +impl FallibleColorSyntax for MemberShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let bare = color_fallible_syntax_with( + &BareShape, + &FlatShape::BareMember, + token_nodes, + context, + shapes, + ); + + match bare { + Ok(_) => return Ok(()), + Err(_) => { + // If we don't have a bare word, we'll look for a string + } + } + + // Look for a string token. If we don't find one, fail + color_fallible_syntax_with( + &StringShape, + &FlatShape::StringMember, + token_nodes, + context, + shapes, + ) + } +} + impl ExpandSyntax for MemberShape { type Output = Member; @@ -317,6 +571,34 @@ impl ExpandSyntax for MemberShape { #[derive(Debug, Copy, Clone)] pub struct DotShape; +#[derive(Debug, Copy, Clone)] +pub struct ColorableDotShape; + +impl FallibleColorSyntax for ColorableDotShape { + type Info = (); + type Input = FlatShape; + + fn color_syntax<'a, 'b>( + &self, + input: &FlatShape, + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let peeked = token_nodes.peek_any().not_eof("dot")?; + + match peeked.node { + node if node.is_dot() => { + peeked.commit(); + shapes.push((*input).tagged(node.tag())); + Ok(()) + } + + other => Err(ShellError::type_error("dot", other.tagged_type_name())), + } + } +} + impl SkipSyntax for DotShape { fn skip<'a, 'b>( &self, @@ -337,7 +619,7 @@ impl ExpandSyntax for DotShape { token_nodes: &'b mut TokensIterator<'a>, _context: &ExpandContext, ) -> Result { - parse_single_node(token_nodes, "dot", |token, token_tag| { + parse_single_node(token_nodes, "dot", |token, token_tag, _| { Ok(match token { RawToken::Operator(Operator::Dot) => token_tag, _ => { @@ -354,6 +636,53 @@ impl ExpandSyntax for DotShape { #[derive(Debug, Copy, Clone)] pub struct InfixShape; +impl FallibleColorSyntax for InfixShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + outer_shapes: &mut Vec>, + ) -> Result<(), ShellError> { + let checkpoint = token_nodes.checkpoint(); + let mut shapes = vec![]; + + // An infix operator must be prefixed by whitespace. If no whitespace was found, fail + color_fallible_syntax(&WhitespaceShape, checkpoint.iterator, context, &mut shapes)?; + + // Parse the next TokenNode after the whitespace + parse_single_node( + checkpoint.iterator, + "infix operator", + |token, token_tag, _| { + match token { + // If it's an operator (and not `.`), it's a match + RawToken::Operator(operator) if operator != Operator::Dot => { + shapes.push(FlatShape::Operator.tagged(token_tag)); + Ok(()) + } + + // Otherwise, it's not a match + _ => Err(ShellError::type_error( + "infix operator", + token.type_name().tagged(token_tag), + )), + } + }, + )?; + + // An infix operator must be followed by whitespace. If no whitespace was found, fail + color_fallible_syntax(&WhitespaceShape, checkpoint.iterator, context, &mut shapes)?; + + outer_shapes.extend(shapes); + checkpoint.commit(); + Ok(()) + } +} + impl ExpandSyntax for InfixShape { type Output = (Tag, Tagged, Tag); @@ -368,8 +697,10 @@ impl ExpandSyntax for InfixShape { let start = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?; // Parse the next TokenNode after the whitespace - let operator = - parse_single_node(checkpoint.iterator, "infix operator", |token, token_tag| { + let operator = parse_single_node( + checkpoint.iterator, + "infix operator", + |token, token_tag, _| { Ok(match token { // If it's an operator (and not `.`), it's a match RawToken::Operator(operator) if operator != Operator::Dot => { @@ -384,7 +715,8 @@ impl ExpandSyntax for InfixShape { )) } }) - })?; + }, + )?; // An infix operator must be followed by whitespace let end = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?; diff --git a/src/parser/hir/syntax_shape/flat_shape.rs b/src/parser/hir/syntax_shape/flat_shape.rs new file mode 100644 index 0000000000..48e867199e --- /dev/null +++ b/src/parser/hir/syntax_shape/flat_shape.rs @@ -0,0 +1,95 @@ +use crate::parser::{Delimiter, Flag, FlagKind, Operator, RawNumber, RawToken, TokenNode}; +use crate::{Tag, Tagged, TaggedItem, Text}; + +#[derive(Debug, Copy, Clone)] +pub enum FlatShape { + OpenDelimiter(Delimiter), + CloseDelimiter(Delimiter), + ItVariable, + Variable, + Operator, + Dot, + InternalCommand, + ExternalCommand, + ExternalWord, + BareMember, + StringMember, + String, + Path, + Word, + Pipe, + GlobPattern, + Flag, + ShorthandFlag, + Int, + Decimal, + Whitespace, + Error, + Size { number: Tag, unit: Tag }, +} + +impl FlatShape { + pub fn from(token: &TokenNode, source: &Text, shapes: &mut Vec>) -> () { + match token { + TokenNode::Token(token) => match token.item { + RawToken::Number(RawNumber::Int(_)) => { + shapes.push(FlatShape::Int.tagged(token.tag)) + } + RawToken::Number(RawNumber::Decimal(_)) => { + shapes.push(FlatShape::Decimal.tagged(token.tag)) + } + RawToken::Operator(Operator::Dot) => shapes.push(FlatShape::Dot.tagged(token.tag)), + RawToken::Operator(_) => shapes.push(FlatShape::Operator.tagged(token.tag)), + RawToken::String(_) => shapes.push(FlatShape::String.tagged(token.tag)), + RawToken::Variable(v) if v.slice(source) == "it" => { + shapes.push(FlatShape::ItVariable.tagged(token.tag)) + } + RawToken::Variable(_) => shapes.push(FlatShape::Variable.tagged(token.tag)), + RawToken::ExternalCommand(_) => { + shapes.push(FlatShape::ExternalCommand.tagged(token.tag)) + } + RawToken::ExternalWord => shapes.push(FlatShape::ExternalWord.tagged(token.tag)), + RawToken::GlobPattern => shapes.push(FlatShape::GlobPattern.tagged(token.tag)), + RawToken::Bare => shapes.push(FlatShape::Word.tagged(token.tag)), + }, + TokenNode::Call(_) => unimplemented!(), + TokenNode::Nodes(nodes) => { + for node in &nodes.item { + FlatShape::from(node, source, shapes); + } + } + TokenNode::Delimited(v) => { + shapes.push(FlatShape::OpenDelimiter(v.item.delimiter).tagged(v.item.tags.0)); + for token in &v.item.children { + FlatShape::from(token, source, shapes); + } + shapes.push(FlatShape::CloseDelimiter(v.item.delimiter).tagged(v.item.tags.1)); + } + TokenNode::Pipeline(pipeline) => { + for part in &pipeline.parts { + if let Some(_) = part.pipe { + shapes.push(FlatShape::Pipe.tagged(part.tag)); + } + } + } + TokenNode::Flag(Tagged { + item: + Flag { + kind: FlagKind::Longhand, + .. + }, + tag, + }) => shapes.push(FlatShape::Flag.tagged(tag)), + TokenNode::Flag(Tagged { + item: + Flag { + kind: FlagKind::Shorthand, + .. + }, + tag, + }) => shapes.push(FlatShape::ShorthandFlag.tagged(tag)), + TokenNode::Whitespace(_) => shapes.push(FlatShape::Whitespace.tagged(token.tag())), + TokenNode::Error(v) => shapes.push(FlatShape::Error.tagged(v.tag)), + } + } +} diff --git a/src/parser/hir/tokens_iterator.rs b/src/parser/hir/tokens_iterator.rs index c0dd9c50fd..f597c850bd 100644 --- a/src/parser/hir/tokens_iterator.rs +++ b/src/parser/hir/tokens_iterator.rs @@ -3,16 +3,13 @@ pub(crate) mod debug; use crate::errors::ShellError; use crate::parser::TokenNode; use crate::{Tag, Tagged, TaggedItem}; -use derive_new::new; -#[derive(Debug, new)] -pub struct TokensIterator<'a> { - tokens: &'a [TokenNode], +#[derive(Debug)] +pub struct TokensIterator<'content> { + tokens: &'content [TokenNode], tag: Tag, skip_ws: bool, - #[new(default)] index: usize, - #[new(default)] seen: indexmap::IndexSet, } @@ -124,11 +121,41 @@ pub fn peek_error( } impl<'content> TokensIterator<'content> { - #[cfg(test)] + pub fn new(items: &'content [TokenNode], tag: Tag, skip_ws: bool) -> TokensIterator<'content> { + TokensIterator { + tokens: items, + tag, + skip_ws, + index: 0, + seen: indexmap::IndexSet::new(), + } + } + + pub fn anchor(&self) -> uuid::Uuid { + self.tag.anchor + } + pub fn all(tokens: &'content [TokenNode], tag: Tag) -> TokensIterator<'content> { TokensIterator::new(tokens, tag, false) } + pub fn len(&self) -> usize { + self.tokens.len() + } + + pub fn spanned( + &mut self, + block: impl FnOnce(&mut TokensIterator<'content>) -> T, + ) -> Tagged { + let start = self.tag_at_cursor(); + + let result = block(self); + + let end = self.tag_at_cursor(); + + result.tagged(start.until(end)) + } + /// Use a checkpoint when you need to peek more than one token ahead, but can't be sure /// that you'll succeed. pub fn checkpoint<'me>(&'me mut self) -> Checkpoint<'content, 'me> { @@ -143,8 +170,26 @@ impl<'content> TokensIterator<'content> { } } - pub fn anchor(&self) -> uuid::Uuid { - self.tag.anchor + /// Use a checkpoint when you need to peek more than one token ahead, but can't be sure + /// that you'll succeed. + pub fn atomic<'me, T>( + &'me mut self, + block: impl FnOnce(&mut TokensIterator<'content>) -> Result, + ) -> Result { + let index = self.index; + let seen = self.seen.clone(); + + let checkpoint = Checkpoint { + iterator: self, + index, + seen, + committed: false, + }; + + let value = block(checkpoint.iterator)?; + + checkpoint.commit(); + return Ok(value); } fn eof_tag(&self) -> Tag { @@ -160,6 +205,15 @@ impl<'content> TokensIterator<'content> { } } + pub fn tag_at_cursor(&mut self) -> Tag { + let next = self.peek_any(); + + match next.node { + None => self.eof_tag(), + Some(node) => node.tag(), + } + } + pub fn remove(&mut self, position: usize) { self.seen.insert(position); } @@ -231,6 +285,26 @@ impl<'content> TokensIterator<'content> { start_next(self, false) } + // Peek the next token, including whitespace, but not EOF + pub fn peek_any_token<'me, T>( + &'me mut self, + block: impl FnOnce(&'content TokenNode) -> Result, + ) -> Result { + let peeked = start_next(self, false); + let peeked = peeked.not_eof("invariant"); + + match peeked { + Err(err) => return Err(err), + Ok(peeked) => match block(peeked.node) { + Err(err) => return Err(err), + Ok(val) => { + peeked.commit(); + return Ok(val); + } + }, + } + } + fn commit(&mut self, from: usize, to: usize) { for index in from..to { self.seen.insert(index); @@ -239,6 +313,10 @@ impl<'content> TokensIterator<'content> { self.index = to; } + pub fn pos(&self, skip_ws: bool) -> Option { + peek_pos(self, skip_ws) + } + pub fn debug_remaining(&self) -> Vec { let mut tokens = self.clone(); tokens.restart(); @@ -246,18 +324,18 @@ impl<'content> TokensIterator<'content> { } } -impl<'a> Iterator for TokensIterator<'a> { - type Item = &'a TokenNode; +impl<'content> Iterator for TokensIterator<'content> { + type Item = &'content TokenNode; - fn next(&mut self) -> Option<&'a TokenNode> { + fn next(&mut self) -> Option<&'content TokenNode> { next(self, self.skip_ws) } } fn peek<'content, 'me>( - iterator: &TokensIterator<'content>, + iterator: &'me TokensIterator<'content>, skip_ws: bool, -) -> Option<&'content TokenNode> { +) -> Option<&'me TokenNode> { let mut to = iterator.index; loop { @@ -287,6 +365,37 @@ fn peek<'content, 'me>( } } +fn peek_pos<'content, 'me>( + iterator: &'me TokensIterator<'content>, + skip_ws: bool, +) -> Option { + let mut to = iterator.index; + + loop { + if to >= iterator.tokens.len() { + return None; + } + + if iterator.seen.contains(&to) { + to += 1; + continue; + } + + if to >= iterator.tokens.len() { + return None; + } + + let node = &iterator.tokens[to]; + + match node { + TokenNode::Whitespace(_) if skip_ws => { + to += 1; + } + _ => return Some(to), + } + } +} + fn start_next<'content, 'me>( iterator: &'me mut TokensIterator<'content>, skip_ws: bool, @@ -337,7 +446,10 @@ fn start_next<'content, 'me>( } } -fn next<'a>(iterator: &mut TokensIterator<'a>, skip_ws: bool) -> Option<&'a TokenNode> { +fn next<'me, 'content>( + iterator: &'me mut TokensIterator<'content>, + skip_ws: bool, +) -> Option<&'content TokenNode> { loop { if iterator.index >= iterator.tokens.len() { return None; diff --git a/src/parser/parse/flag.rs b/src/parser/parse/flag.rs index 09d1e86337..b8995305d2 100644 --- a/src/parser/parse/flag.rs +++ b/src/parser/parse/flag.rs @@ -1,4 +1,5 @@ -use crate::Tag; +use crate::parser::hir::syntax_shape::flat_shape::FlatShape; +use crate::{Tag, Tagged, TaggedItem}; use derive_new::new; use getset::Getters; use serde::{Deserialize, Serialize}; @@ -12,6 +13,15 @@ pub enum FlagKind { #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Getters, new)] #[get = "pub(crate)"] pub struct Flag { - kind: FlagKind, - name: Tag, + pub(crate) kind: FlagKind, + pub(crate) name: Tag, +} + +impl Tagged { + pub fn color(&self) -> Tagged { + match self.item.kind { + FlagKind::Longhand => FlatShape::Flag.tagged(self.tag), + FlagKind::Shorthand => FlatShape::ShorthandFlag.tagged(self.tag), + } + } } diff --git a/src/parser/parse/parser.rs b/src/parser/parse/parser.rs index 93ba043ba1..73833f7be5 100644 --- a/src/parser/parse/parser.rs +++ b/src/parser/parse/parser.rs @@ -189,7 +189,7 @@ pub fn raw_number(input: NomSpan) -> IResult> { match input.fragment.chars().next() { None => return Ok((input, RawNumber::int((start, input.offset, input.extra)))), Some('.') => (), - Some(other) if other.is_whitespace() => { + other if is_boundary(other) => { return Ok((input, RawNumber::int((start, input.offset, input.extra)))) } _ => { @@ -215,16 +215,14 @@ pub fn raw_number(input: NomSpan) -> IResult> { let next = input.fragment.chars().next(); - if let Some(next) = next { - if !next.is_whitespace() { - return Err(nom::Err::Error(nom::error::make_error( - input, - nom::error::ErrorKind::Tag, - ))); - } + if is_boundary(next) { + Ok((input, RawNumber::decimal((start, end, input.extra)))) + } else { + Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::Tag, + ))) } - - Ok((input, RawNumber::decimal((start, end, input.extra)))) } #[tracable_parser] @@ -476,11 +474,14 @@ pub fn whitespace(input: NomSpan) -> IResult { )) } -pub fn delimited(input: NomSpan, delimiter: Delimiter) -> IResult>> { +pub fn delimited( + input: NomSpan, + delimiter: Delimiter, +) -> IResult>)> { let left = input.offset; - let (input, _) = char(delimiter.open())(input)?; + let (input, open_tag) = tag(delimiter.open())(input)?; let (input, inner_items) = opt(spaced_token_list)(input)?; - let (input, _) = char(delimiter.close())(input)?; + let (input, close_tag) = tag(delimiter.close())(input)?; let right = input.offset; let mut items = vec![]; @@ -489,36 +490,43 @@ pub fn delimited(input: NomSpan, delimiter: Delimiter) -> IResult IResult { - let (input, tokens) = delimited(input, Delimiter::Paren)?; + let (input, (left, right, tokens)) = delimited(input, Delimiter::Paren)?; Ok(( input, - TokenTreeBuilder::tagged_parens(tokens.item, tokens.tag), + TokenTreeBuilder::tagged_parens(tokens.item, (left, right), tokens.tag), )) } #[tracable_parser] pub fn delimited_square(input: NomSpan) -> IResult { - let (input, tokens) = delimited(input, Delimiter::Square)?; + let (input, (left, right, tokens)) = delimited(input, Delimiter::Square)?; Ok(( input, - TokenTreeBuilder::tagged_square(tokens.item, tokens.tag), + TokenTreeBuilder::tagged_square(tokens.item, (left, right), tokens.tag), )) } #[tracable_parser] pub fn delimited_brace(input: NomSpan) -> IResult { - let (input, tokens) = delimited(input, Delimiter::Brace)?; + let (input, (left, right, tokens)) = delimited(input, Delimiter::Brace)?; Ok(( input, - TokenTreeBuilder::tagged_brace(tokens.item, tokens.tag), + TokenTreeBuilder::tagged_square(tokens.item, (left, right), tokens.tag), )) } @@ -1246,7 +1254,10 @@ mod tests { left: usize, right: usize, ) -> TokenNode { - let node = DelimitedNode::new(*delimiter, children); + let start = Tag::for_char(left, delimiter.tag.anchor); + let end = Tag::for_char(right, delimiter.tag.anchor); + + let node = DelimitedNode::new(delimiter.item, (start, end), children); let spanned = node.tagged((left, right, delimiter.tag.anchor)); TokenNode::Delimited(spanned) } diff --git a/src/parser/parse/token_tree.rs b/src/parser/parse/token_tree.rs index 8cbb28264b..85961d1dab 100644 --- a/src/parser/parse/token_tree.rs +++ b/src/parser/parse/token_tree.rs @@ -1,5 +1,5 @@ use crate::errors::ShellError; -use crate::parser::parse::{call_node::*, flag::*, pipeline::*, tokens::*}; +use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*}; use crate::prelude::*; use crate::traits::ToDebug; use crate::{Tag, Tagged, Text}; @@ -17,10 +17,9 @@ pub enum TokenNode { Delimited(Tagged), Pipeline(Tagged), Flag(Tagged), - Member(Tag), Whitespace(Tag), - Error(Tagged>), + Error(Tagged), } impl ToDebug for TokenNode { @@ -78,7 +77,7 @@ impl fmt::Debug for DebugTokenNode<'_> { ) } TokenNode::Pipeline(pipeline) => write!(f, "{}", pipeline.debug(self.source)), - TokenNode::Error(s) => write!(f, " for {:?}", s.tag().slice(self.source)), + TokenNode::Error(_) => write!(f, ""), rest => write!(f, "{}", rest.tag().slice(self.source)), } } @@ -99,9 +98,8 @@ impl TokenNode { TokenNode::Delimited(s) => s.tag(), TokenNode::Pipeline(s) => s.tag(), TokenNode::Flag(s) => s.tag(), - TokenNode::Member(s) => *s, TokenNode::Whitespace(s) => *s, - TokenNode::Error(s) => s.tag(), + TokenNode::Error(s) => return s.tag, } } @@ -113,7 +111,6 @@ impl TokenNode { TokenNode::Delimited(d) => d.type_name(), TokenNode::Pipeline(_) => "pipeline", TokenNode::Flag(_) => "flag", - TokenNode::Member(_) => "member", TokenNode::Whitespace(_) => "whitespace", TokenNode::Error(_) => "error", } @@ -155,16 +152,37 @@ impl TokenNode { } } - pub fn as_block(&self) -> Option> { + pub fn is_pattern(&self) -> bool { + match self { + TokenNode::Token(Tagged { + item: RawToken::GlobPattern, + .. + }) => true, + _ => false, + } + } + + pub fn is_dot(&self) -> bool { + match self { + TokenNode::Token(Tagged { + item: RawToken::Operator(Operator::Dot), + .. + }) => true, + _ => false, + } + } + + pub fn as_block(&self) -> Option<(Tagged<&[TokenNode]>, (Tag, Tag))> { match self { TokenNode::Delimited(Tagged { item: DelimitedNode { delimiter, children, + tags, }, tag, - }) if *delimiter == Delimiter::Brace => Some((&children[..]).tagged(tag)), + }) if *delimiter == Delimiter::Brace => Some(((&children[..]).tagged(tag), *tags)), _ => None, } } @@ -203,7 +221,7 @@ impl TokenNode { pub fn as_pipeline(&self) -> Result { match self { TokenNode::Pipeline(Tagged { item, .. }) => Ok(item.clone()), - _ => Err(ShellError::string("unimplemented")), + _ => Err(ShellError::unimplemented("unimplemented")), } } @@ -259,6 +277,7 @@ impl TokenNode { #[get = "pub(crate)"] pub struct DelimitedNode { pub(crate) delimiter: Delimiter, + pub(crate) tags: (Tag, Tag), pub(crate) children: Vec, } @@ -280,19 +299,19 @@ pub enum Delimiter { } impl Delimiter { - pub(crate) fn open(&self) -> char { + pub(crate) fn open(&self) -> &'static str { match self { - Delimiter::Paren => '(', - Delimiter::Brace => '{', - Delimiter::Square => '[', + Delimiter::Paren => "(", + Delimiter::Brace => "{", + Delimiter::Square => "[", } } - pub(crate) fn close(&self) -> char { + pub(crate) fn close(&self) -> &'static str { match self { - Delimiter::Paren => ')', - Delimiter::Brace => '}', - Delimiter::Square => ']', + Delimiter::Paren => ")", + Delimiter::Brace => "}", + Delimiter::Square => "]", } } } diff --git a/src/parser/parse/token_tree_builder.rs b/src/parser/parse/token_tree_builder.rs index 67298987a4..549462a979 100644 --- a/src/parser/parse/token_tree_builder.rs +++ b/src/parser/parse/token_tree_builder.rs @@ -5,7 +5,6 @@ use crate::parser::parse::operator::Operator; use crate::parser::parse::pipeline::{Pipeline, PipelineElement}; use crate::parser::parse::token_tree::{DelimitedNode, Delimiter, TokenNode}; use crate::parser::parse::tokens::{RawNumber, RawToken}; -use crate::parser::parse::unit::Unit; use crate::parser::CallNode; use derive_new::new; use uuid::Uuid; @@ -227,31 +226,6 @@ impl TokenTreeBuilder { TokenNode::Token(RawToken::Number(input.into()).tagged(tag.into())) } - pub fn size(int: impl Into, unit: impl Into) -> CurriedToken { - let int = int.into(); - let unit = unit.into(); - - Box::new(move |b| { - let (start_int, end_int) = b.consume(&int.to_string()); - let (_, end_unit) = b.consume(unit.as_str()); - b.pos = end_unit; - - TokenTreeBuilder::tagged_size( - (RawNumber::Int((start_int, end_int, b.anchor).into()), unit), - (start_int, end_unit, b.anchor), - ) - }) - } - - pub fn tagged_size( - input: (impl Into, impl Into), - tag: impl Into, - ) -> TokenNode { - let (int, unit) = (input.0.into(), input.1.into()); - - TokenNode::Token(RawToken::Size(int, unit).tagged(tag.into())) - } - pub fn var(input: impl Into) -> CurriedToken { let input = input.into(); @@ -297,19 +271,6 @@ impl TokenTreeBuilder { TokenNode::Flag(Flag::new(FlagKind::Shorthand, input.into()).tagged(tag.into())) } - pub fn member(input: impl Into) -> CurriedToken { - let input = input.into(); - - Box::new(move |b| { - let (start, end) = b.consume(&input); - TokenTreeBuilder::tagged_member((start, end, b.anchor)) - }) - } - - pub fn tagged_member(tag: impl Into) -> TokenNode { - TokenNode::Member(tag.into()) - } - pub fn call(head: CurriedToken, input: Vec) -> CurriedCall { Box::new(move |b| { let start = b.pos; @@ -340,58 +301,79 @@ impl TokenTreeBuilder { CallNode::new(Box::new(head), tail).tagged(tag.into()) } + fn consume_delimiter( + &mut self, + input: Vec, + _open: &str, + _close: &str, + ) -> (Tag, Tag, Tag, Vec) { + let (start_open_paren, end_open_paren) = self.consume("("); + let mut output = vec![]; + for item in input { + output.push(item(self)); + } + + let (start_close_paren, end_close_paren) = self.consume(")"); + + let open = Tag::from((start_open_paren, end_open_paren, self.anchor)); + let close = Tag::from((start_close_paren, end_close_paren, self.anchor)); + let whole = Tag::from((start_open_paren, end_close_paren, self.anchor)); + + (open, close, whole, output) + } + pub fn parens(input: Vec) -> CurriedToken { Box::new(move |b| { - let (start, _) = b.consume("("); - let mut output = vec![]; - for item in input { - output.push(item(b)); - } + let (open, close, whole, output) = b.consume_delimiter(input, "(", ")"); - let (_, end) = b.consume(")"); - - TokenTreeBuilder::tagged_parens(output, (start, end, b.anchor)) + TokenTreeBuilder::tagged_parens(output, (open, close), whole) }) } - pub fn tagged_parens(input: impl Into>, tag: impl Into) -> TokenNode { - TokenNode::Delimited(DelimitedNode::new(Delimiter::Paren, input.into()).tagged(tag.into())) + pub fn tagged_parens( + input: impl Into>, + tags: (Tag, Tag), + tag: impl Into, + ) -> TokenNode { + TokenNode::Delimited( + DelimitedNode::new(Delimiter::Paren, tags, input.into()).tagged(tag.into()), + ) } pub fn square(input: Vec) -> CurriedToken { Box::new(move |b| { - let (start, _) = b.consume("["); - let mut output = vec![]; - for item in input { - output.push(item(b)); - } + let (open, close, whole, tokens) = b.consume_delimiter(input, "[", "]"); - let (_, end) = b.consume("]"); - - TokenTreeBuilder::tagged_square(output, (start, end, b.anchor)) + TokenTreeBuilder::tagged_square(tokens, (open, close), whole) }) } - pub fn tagged_square(input: impl Into>, tag: impl Into) -> TokenNode { - TokenNode::Delimited(DelimitedNode::new(Delimiter::Square, input.into()).tagged(tag.into())) + pub fn tagged_square( + input: impl Into>, + tags: (Tag, Tag), + tag: impl Into, + ) -> TokenNode { + TokenNode::Delimited( + DelimitedNode::new(Delimiter::Square, tags, input.into()).tagged(tag.into()), + ) } pub fn braced(input: Vec) -> CurriedToken { Box::new(move |b| { - let (start, _) = b.consume("{ "); - let mut output = vec![]; - for item in input { - output.push(item(b)); - } + let (open, close, whole, tokens) = b.consume_delimiter(input, "{", "}"); - let (_, end) = b.consume(" }"); - - TokenTreeBuilder::tagged_brace(output, (start, end, b.anchor)) + TokenTreeBuilder::tagged_brace(tokens, (open, close), whole) }) } - pub fn tagged_brace(input: impl Into>, tag: impl Into) -> TokenNode { - TokenNode::Delimited(DelimitedNode::new(Delimiter::Brace, input.into()).tagged(tag.into())) + pub fn tagged_brace( + input: impl Into>, + tags: (Tag, Tag), + tag: impl Into, + ) -> TokenNode { + TokenNode::Delimited( + DelimitedNode::new(Delimiter::Brace, tags, input.into()).tagged(tag.into()), + ) } pub fn sp() -> CurriedToken { diff --git a/src/parser/parse/tokens.rs b/src/parser/parse/tokens.rs index 77a856af3f..41bdfcebd6 100644 --- a/src/parser/parse/tokens.rs +++ b/src/parser/parse/tokens.rs @@ -1,4 +1,3 @@ -use crate::parser::parse::unit::*; use crate::parser::Operator; use crate::prelude::*; use crate::{Tagged, Text}; @@ -9,7 +8,6 @@ use std::str::FromStr; pub enum RawToken { Number(RawNumber), Operator(Operator), - Size(RawNumber, Unit), String(Tag), Variable(Tag), ExternalCommand(Tag), @@ -18,6 +16,21 @@ pub enum RawToken { Bare, } +impl RawToken { + pub fn type_name(&self) -> &'static str { + match self { + RawToken::Number(_) => "Number", + RawToken::Operator(..) => "operator", + RawToken::String(_) => "String", + RawToken::Variable(_) => "variable", + RawToken::ExternalCommand(_) => "external command", + RawToken::ExternalWord => "external word", + RawToken::GlobPattern => "glob pattern", + RawToken::Bare => "String", + } + } +} + #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum RawNumber { Int(Tag), @@ -47,22 +60,6 @@ impl RawNumber { } } -impl RawToken { - pub fn type_name(&self) -> &'static str { - match self { - RawToken::Number(_) => "Number", - RawToken::Operator(..) => "operator", - RawToken::Size(..) => "Size", - RawToken::String(_) => "String", - RawToken::Variable(_) => "variable", - RawToken::ExternalCommand(_) => "external command", - RawToken::ExternalWord => "external word", - RawToken::GlobPattern => "glob pattern", - RawToken::Bare => "String", - } - } -} - pub type Token = Tagged; impl Token { @@ -72,6 +69,76 @@ impl Token { source, } } + + pub fn extract_number(&self) -> Option> { + match self.item { + RawToken::Number(number) => Some((number).tagged(self.tag)), + _ => None, + } + } + + pub fn extract_int(&self) -> Option<(Tag, Tag)> { + match self.item { + RawToken::Number(RawNumber::Int(int)) => Some((int, self.tag)), + _ => None, + } + } + + pub fn extract_decimal(&self) -> Option<(Tag, Tag)> { + match self.item { + RawToken::Number(RawNumber::Decimal(decimal)) => Some((decimal, self.tag)), + _ => None, + } + } + + pub fn extract_operator(&self) -> Option> { + match self.item { + RawToken::Operator(operator) => Some(operator.tagged(self.tag)), + _ => None, + } + } + + pub fn extract_string(&self) -> Option<(Tag, Tag)> { + match self.item { + RawToken::String(tag) => Some((tag, self.tag)), + _ => None, + } + } + + pub fn extract_variable(&self) -> Option<(Tag, Tag)> { + match self.item { + RawToken::Variable(tag) => Some((tag, self.tag)), + _ => None, + } + } + + pub fn extract_external_command(&self) -> Option<(Tag, Tag)> { + match self.item { + RawToken::ExternalCommand(tag) => Some((tag, self.tag)), + _ => None, + } + } + + pub fn extract_external_word(&self) -> Option { + match self.item { + RawToken::ExternalWord => Some(self.tag), + _ => None, + } + } + + pub fn extract_glob_pattern(&self) -> Option { + match self.item { + RawToken::GlobPattern => Some(self.tag), + _ => None, + } + } + + pub fn extract_bare(&self) -> Option { + match self.item { + RawToken::Bare => Some(self.tag), + _ => None, + } + } } pub struct DebugToken<'a> { diff --git a/src/parser/parse_command.rs b/src/parser/parse_command.rs index d383689fd9..603ff2956d 100644 --- a/src/parser/parse_command.rs +++ b/src/parser/parse_command.rs @@ -1,5 +1,8 @@ use crate::errors::{ArgumentError, ShellError}; -use crate::parser::hir::syntax_shape::{expand_expr, spaced}; +use crate::parser::hir::syntax_shape::{ + color_fallible_syntax, color_syntax, expand_expr, flat_shape::FlatShape, spaced, + BackoffColoringMode, ColorSyntax, MaybeSpaceShape, +}; use crate::parser::registry::{NamedType, PositionalType, Signature}; use crate::parser::TokensIterator; use crate::parser::{ @@ -153,6 +156,232 @@ pub fn parse_command_tail( Ok(Some((positional, named))) } +#[derive(Debug)] +struct ColoringArgs { + vec: Vec>>>, +} + +impl ColoringArgs { + fn new(len: usize) -> ColoringArgs { + let vec = vec![None; len]; + ColoringArgs { vec } + } + + fn insert(&mut self, pos: usize, shapes: Vec>) { + self.vec[pos] = Some(shapes); + } + + fn spread_shapes(self, shapes: &mut Vec>) { + for item in self.vec { + match item { + None => {} + Some(vec) => { + shapes.extend(vec); + } + } + } + } +} + +#[derive(Debug, Copy, Clone)] +pub struct CommandTailShape; + +impl ColorSyntax for CommandTailShape { + type Info = (); + type Input = Signature; + + fn color_syntax<'a, 'b>( + &self, + signature: &Signature, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Self::Info { + let mut args = ColoringArgs::new(token_nodes.len()); + trace_remaining("nodes", token_nodes.clone(), context.source()); + + for (name, kind) in &signature.named { + trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind); + + match kind { + NamedType::Switch => { + match token_nodes.extract(|t| t.as_flag(name, context.source())) { + Some((pos, flag)) => args.insert(pos, vec![flag.color()]), + None => {} + } + } + NamedType::Mandatory(syntax_type) => { + match extract_mandatory( + signature, + name, + token_nodes, + context.source(), + Tag::unknown(), + ) { + Err(_) => { + // The mandatory flag didn't exist at all, so there's nothing to color + } + Ok((pos, flag)) => { + let mut shapes = vec![flag.color()]; + token_nodes.move_to(pos); + + if token_nodes.at_end() { + args.insert(pos, shapes); + token_nodes.restart(); + continue; + } + + // We can live with unmatched syntax after a mandatory flag + let _ = token_nodes.atomic(|token_nodes| { + color_syntax(&MaybeSpaceShape, token_nodes, context, &mut shapes); + + // If the part after a mandatory flag isn't present, that's ok, but we + // should roll back any whitespace we chomped + color_fallible_syntax( + syntax_type, + token_nodes, + context, + &mut shapes, + ) + }); + + args.insert(pos, shapes); + token_nodes.restart(); + } + } + } + NamedType::Optional(syntax_type) => { + match extract_optional(name, token_nodes, context.source()) { + Err(_) => { + // The optional flag didn't exist at all, so there's nothing to color + } + Ok(Some((pos, flag))) => { + let mut shapes = vec![flag.color()]; + token_nodes.move_to(pos); + + if token_nodes.at_end() { + args.insert(pos, shapes); + token_nodes.restart(); + continue; + } + + // We can live with unmatched syntax after an optional flag + let _ = token_nodes.atomic(|token_nodes| { + color_syntax(&MaybeSpaceShape, token_nodes, context, &mut shapes); + + // If the part after a mandatory flag isn't present, that's ok, but we + // should roll back any whitespace we chomped + color_fallible_syntax( + syntax_type, + token_nodes, + context, + &mut shapes, + ) + }); + + args.insert(pos, shapes); + token_nodes.restart(); + } + + Ok(None) => { + token_nodes.restart(); + } + } + } + }; + } + + trace_remaining("after named", token_nodes.clone(), context.source()); + + for arg in &signature.positional { + trace!("Processing positional {:?}", arg); + + match arg { + PositionalType::Mandatory(..) => { + if token_nodes.at_end() { + break; + } + } + + PositionalType::Optional(..) => { + if token_nodes.at_end() { + break; + } + } + } + + let mut shapes = vec![]; + let pos = token_nodes.pos(false); + + match pos { + None => break, + Some(pos) => { + // We can live with an unmatched positional argument. Hopefully it will be + // matched by a future token + let _ = token_nodes.atomic(|token_nodes| { + color_syntax(&MaybeSpaceShape, token_nodes, context, &mut shapes); + + // If no match, we should roll back any whitespace we chomped + color_fallible_syntax( + &arg.syntax_type(), + token_nodes, + context, + &mut shapes, + )?; + + args.insert(pos, shapes); + + Ok(()) + }); + } + } + } + + trace_remaining("after positional", token_nodes.clone(), context.source()); + + if let Some(syntax_type) = signature.rest_positional { + loop { + if token_nodes.at_end_possible_ws() { + break; + } + + let pos = token_nodes.pos(false); + + match pos { + None => break, + Some(pos) => { + let mut shapes = vec![]; + + // If any arguments don't match, we'll fall back to backoff coloring mode + let result = token_nodes.atomic(|token_nodes| { + color_syntax(&MaybeSpaceShape, token_nodes, context, &mut shapes); + + // If no match, we should roll back any whitespace we chomped + color_fallible_syntax(&syntax_type, token_nodes, context, &mut shapes)?; + + args.insert(pos, shapes); + + Ok(()) + }); + + match result { + Err(_) => break, + Ok(_) => continue, + } + } + } + } + } + + args.spread_shapes(shapes); + + // Consume any remaining tokens with backoff coloring mode + color_syntax(&BackoffColoringMode, token_nodes, context, shapes); + + shapes.sort_by(|a, b| a.tag.span.start().cmp(&b.tag.span.start())); + } +} + fn extract_switch(name: &str, tokens: &mut hir::TokensIterator<'_>, source: &Text) -> Option { tokens .extract(|t| t.as_flag(name, source)) @@ -200,6 +429,7 @@ fn extract_optional( pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'_>, source: &Text) { trace!( + target: "nu::expand_args", "{} = {:?}", desc, itertools::join( diff --git a/src/plugin.rs b/src/plugin.rs index afd9871108..004e937fe8 100644 --- a/src/plugin.rs +++ b/src/plugin.rs @@ -32,7 +32,7 @@ pub fn serve_plugin(plugin: &mut dyn Plugin) { let input = match input { Some(arg) => std::fs::read_to_string(arg), None => { - send_response(ShellError::string(format!("No input given."))); + send_response(ShellError::untagged_runtime_error("No input given.")); return; } }; @@ -64,7 +64,7 @@ pub fn serve_plugin(plugin: &mut dyn Plugin) { return; } e => { - send_response(ShellError::string(format!( + send_response(ShellError::untagged_runtime_error(format!( "Could not handle plugin message: {} {:?}", input, e ))); @@ -102,7 +102,7 @@ pub fn serve_plugin(plugin: &mut dyn Plugin) { break; } e => { - send_response(ShellError::string(format!( + send_response(ShellError::untagged_runtime_error(format!( "Could not handle plugin message: {} {:?}", input, e ))); @@ -111,7 +111,7 @@ pub fn serve_plugin(plugin: &mut dyn Plugin) { } } e => { - send_response(ShellError::string(format!( + send_response(ShellError::untagged_runtime_error(format!( "Could not handle plugin message: {:?}", e, ))); diff --git a/src/plugins/add.rs b/src/plugins/add.rs index 997400d67f..6fc034226c 100644 --- a/src/plugins/add.rs +++ b/src/plugins/add.rs @@ -1,7 +1,7 @@ use itertools::Itertools; use nu::{ serve_plugin, CallInfo, Plugin, ReturnSuccess, ReturnValue, ShellError, Signature, SyntaxShape, - Tagged, Value, + Tagged, TaggedItem, Value, }; pub type ColumnPath = Vec>; @@ -25,21 +25,27 @@ impl Add { Some(f) => match obj.insert_data_at_column_path(value_tag, &f, v) { Some(v) => return Ok(v), None => { - return Err(ShellError::string(format!( - "add could not find place to insert field {:?} {}", - obj, - f.iter().map(|i| &i.item).join(".") - ))) + return Err(ShellError::labeled_error( + format!( + "add could not find place to insert field {:?} {}", + obj, + f.iter().map(|i| &i.item).join(".") + ), + "column name", + value_tag, + )) } }, - None => Err(ShellError::string( + None => Err(ShellError::labeled_error( "add needs a column name when adding a value to a table", + "column name", + value_tag, )), }, - x => Err(ShellError::string(format!( - "Unrecognized type in stream: {:?}", - x - ))), + (value, _) => Err(ShellError::type_error( + "row", + value.type_name().tagged(value_tag), + )), } } } @@ -64,12 +70,7 @@ impl Plugin for Add { self.field = Some(table.as_column_path()?.item); } - _ => { - return Err(ShellError::string(format!( - "Unrecognized type in params: {:?}", - args[0] - ))) - } + value => return Err(ShellError::type_error("table", value.tagged_type_name())), } match &args[1] { Tagged { item: v, .. } => { diff --git a/src/plugins/edit.rs b/src/plugins/edit.rs index 6d35530ef5..c0f6dfbedd 100644 --- a/src/plugins/edit.rs +++ b/src/plugins/edit.rs @@ -3,7 +3,7 @@ use nu::{ Tagged, Value, }; -pub type ColumnPath = Vec>; +pub type ColumnPath = Tagged>>; struct Edit { field: Option, @@ -24,19 +24,22 @@ impl Edit { Some(f) => match obj.replace_data_at_column_path(value_tag, &f, v) { Some(v) => return Ok(v), None => { - return Err(ShellError::string( + return Err(ShellError::labeled_error( "edit could not find place to insert column", + "column name", + f.tag, )) } }, - None => Err(ShellError::string( + None => Err(ShellError::untagged_runtime_error( "edit needs a column when changing a value in a table", )), }, - x => Err(ShellError::string(format!( - "Unrecognized type in stream: {:?}", - x - ))), + _ => Err(ShellError::labeled_error( + "Unrecognized type in stream", + "original value", + value_tag, + )), } } } @@ -57,14 +60,9 @@ impl Plugin for Edit { item: Value::Table(_), .. } => { - self.field = Some(table.as_column_path()?.item); - } - _ => { - return Err(ShellError::string(format!( - "Unrecognized type in params: {:?}", - args[0] - ))) + self.field = Some(table.as_column_path()?); } + value => return Err(ShellError::type_error("table", value.tagged_type_name())), } match &args[1] { Tagged { item: v, .. } => { diff --git a/src/plugins/embed.rs b/src/plugins/embed.rs index 646db80918..4e3545d055 100644 --- a/src/plugins/embed.rs +++ b/src/plugins/embed.rs @@ -25,8 +25,10 @@ impl Embed { }); Ok(()) } - None => Err(ShellError::string( + None => Err(ShellError::labeled_error( "embed needs a field when embedding a value", + "original value", + value.tag, )), }, } @@ -52,12 +54,7 @@ impl Plugin for Embed { self.field = Some(s.clone()); self.values = Vec::new(); } - _ => { - return Err(ShellError::string(format!( - "Unrecognized type in params: {:?}", - args[0] - ))) - } + value => return Err(ShellError::type_error("string", value.tagged_type_name())), } } diff --git a/src/plugins/inc.rs b/src/plugins/inc.rs index 4e6f6f0f64..c58ca89369 100644 --- a/src/plugins/inc.rs +++ b/src/plugins/inc.rs @@ -14,7 +14,7 @@ pub enum SemVerAction { Patch, } -pub type ColumnPath = Vec>; +pub type ColumnPath = Tagged>>; struct Inc { field: Option, @@ -90,7 +90,11 @@ impl Inc { let replacement = match value.item.get_data_by_column_path(value.tag(), f) { Some(result) => self.inc(result.map(|x| x.clone()))?, None => { - return Err(ShellError::string("inc could not find field to replace")) + return Err(ShellError::labeled_error( + "inc could not find field to replace", + "column name", + f.tag, + )) } }; match value.item.replace_data_at_column_path( @@ -100,18 +104,22 @@ impl Inc { ) { Some(v) => return Ok(v), None => { - return Err(ShellError::string("inc could not find field to replace")) + return Err(ShellError::labeled_error( + "inc could not find field to replace", + "column name", + f.tag, + )) } } } - None => Err(ShellError::string( + None => Err(ShellError::untagged_runtime_error( "inc needs a field when incrementing a column in a table", )), }, - x => Err(ShellError::string(format!( - "Unrecognized type in stream: {:?}", - x - ))), + _ => Err(ShellError::type_error( + "incrementable value", + value.tagged_type_name(), + )), } } } @@ -145,14 +153,9 @@ impl Plugin for Inc { item: Value::Table(_), .. } => { - self.field = Some(table.as_column_path()?.item); - } - _ => { - return Err(ShellError::string(format!( - "Unrecognized type in params: {:?}", - arg - ))) + self.field = Some(table.as_column_path()?); } + value => return Err(ShellError::type_error("table", value.tagged_type_name())), } } } @@ -163,7 +166,11 @@ impl Plugin for Inc { match &self.error { Some(reason) => { - return Err(ShellError::string(format!("{}: {}", reason, Inc::usage()))) + return Err(ShellError::untagged_runtime_error(format!( + "{}: {}", + reason, + Inc::usage() + ))) } None => Ok(vec![]), } @@ -308,7 +315,7 @@ mod tests { assert_eq!( plugin .field - .map(|f| f.into_iter().map(|f| f.item).collect()), + .map(|f| f.iter().map(|f| f.item.clone()).collect()), Some(vec!["package".to_string(), "version".to_string()]) ); } diff --git a/src/plugins/match.rs b/src/plugins/match.rs index 1f2aad83fc..7133524050 100644 --- a/src/plugins/match.rs +++ b/src/plugins/match.rs @@ -35,11 +35,12 @@ impl Plugin for Match { } => { self.column = s.clone(); } - _ => { - return Err(ShellError::string(format!( - "Unrecognized type in params: {:?}", - args[0] - ))); + Tagged { tag, .. } => { + return Err(ShellError::labeled_error( + "Unrecognized type in params", + "value", + tag, + )); } } match &args[1] { @@ -49,11 +50,12 @@ impl Plugin for Match { } => { self.regex = Regex::new(s).unwrap(); } - _ => { - return Err(ShellError::string(format!( - "Unrecognized type in params: {:?}", - args[1] - ))); + Tagged { tag, .. } => { + return Err(ShellError::labeled_error( + "Unrecognized type in params", + "value", + tag, + )); } } } @@ -65,7 +67,7 @@ impl Plugin for Match { match &input { Tagged { item: Value::Row(dict), - .. + tag, } => { if let Some(val) = dict.entries.get(&self.column) { match val { @@ -75,22 +77,20 @@ impl Plugin for Match { } => { flag = self.regex.is_match(s); } - _ => { - return Err(ShellError::string(format!( - "value is not a string! {:?}", - &val - ))); + Tagged { tag, .. } => { + return Err(ShellError::labeled_error("expected string", "value", tag)); } } } else { - return Err(ShellError::string(format!( - "column not in row! {:?} {:?}", - &self.column, dict - ))); + return Err(ShellError::labeled_error( + format!("column not in row! {:?} {:?}", &self.column, dict), + "row", + tag, + )); } } - _ => { - return Err(ShellError::string(format!("Not a row! {:?}", &input))); + Tagged { tag, .. } => { + return Err(ShellError::labeled_error("Expected row", "value", tag)); } } if flag { diff --git a/src/plugins/str.rs b/src/plugins/str.rs index 7bd35733da..4635d60c35 100644 --- a/src/plugins/str.rs +++ b/src/plugins/str.rs @@ -105,20 +105,24 @@ impl Str { ) { Some(v) => return Ok(v), None => { - return Err(ShellError::string("str could not find field to replace")) + return Err(ShellError::type_error( + "column name", + value.tagged_type_name(), + )) } } } - None => Err(ShellError::string(format!( + None => Err(ShellError::untagged_runtime_error(format!( "{}: {}", "str needs a column when applied to a value in a row", Str::usage() ))), }, - x => Err(ShellError::string(format!( - "Unrecognized type in stream: {:?}", - x - ))), + _ => Err(ShellError::labeled_error( + "Unrecognized type in stream", + value.type_name(), + value.tag, + )), } } } @@ -167,10 +171,11 @@ impl Plugin for Str { self.field = Some(table.as_column_path()?.item); } _ => { - return Err(ShellError::string(format!( - "Unrecognized type in params: {:?}", - possible_field - ))) + return Err(ShellError::labeled_error( + "Unrecognized type in params", + possible_field.type_name(), + possible_field.tag, + )) } } } @@ -187,7 +192,11 @@ impl Plugin for Str { match &self.error { Some(reason) => { - return Err(ShellError::string(format!("{}: {}", reason, Str::usage()))) + return Err(ShellError::untagged_runtime_error(format!( + "{}: {}", + reason, + Str::usage() + ))) } None => Ok(vec![]), } diff --git a/src/plugins/sum.rs b/src/plugins/sum.rs index ffb39cb90b..2bb89b74e1 100644 --- a/src/plugins/sum.rs +++ b/src/plugins/sum.rs @@ -28,9 +28,11 @@ impl Sum { self.total = Some(value.clone()); Ok(()) } - _ => Err(ShellError::string(format!( - "Could not sum non-integer or unrelated types" - ))), + _ => Err(ShellError::labeled_error( + "Could not sum non-integer or unrelated types", + "source", + value.tag, + )), } } Value::Primitive(Primitive::Bytes(b)) => { @@ -47,15 +49,18 @@ impl Sum { self.total = Some(value); Ok(()) } - _ => Err(ShellError::string(format!( - "Could not sum non-integer or unrelated types" - ))), + _ => Err(ShellError::labeled_error( + "Could not sum non-integer or unrelated types", + "source", + value.tag, + )), } } - x => Err(ShellError::string(format!( - "Unrecognized type in stream: {:?}", - x - ))), + x => Err(ShellError::labeled_error( + format!("Unrecognized type in stream: {:?}", x), + "source", + value.tag, + )), } } } diff --git a/src/prelude.rs b/src/prelude.rs index eabd778717..1f80126a4f 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -1,3 +1,13 @@ +#[macro_export] +macro_rules! return_err { + ($expr:expr) => { + match $expr { + Err(_) => return, + Ok(expr) => expr, + }; + }; +} + #[macro_export] macro_rules! stream { ($($expr:expr),*) => {{ diff --git a/src/shell/filesystem_shell.rs b/src/shell/filesystem_shell.rs index 3c1ae79ea3..aec736ec0f 100644 --- a/src/shell/filesystem_shell.rs +++ b/src/shell/filesystem_shell.rs @@ -145,7 +145,7 @@ impl Shell for FilesystemShell { source.tag(), )); } else { - return Err(ShellError::string("Invalid pattern.")); + return Err(ShellError::untagged_runtime_error("Invalid pattern.")); } } }; diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 85591cf047..b590d82826 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -1,11 +1,11 @@ +use crate::context::Context; +use crate::parser::hir::syntax_shape::{color_fallible_syntax, FlatShape, PipelineShape}; use crate::parser::hir::TokensIterator; use crate::parser::nom_input; use crate::parser::parse::token_tree::TokenNode; -use crate::parser::parse::tokens::RawToken; -use crate::parser::{Pipeline, PipelineElement}; -use crate::shell::shell_manager::ShellManager; -use crate::Tagged; +use crate::{Tag, Tagged, TaggedItem, Text}; use ansi_term::Color; +use log::trace; use rustyline::completion::Completer; use rustyline::error::ReadlineError; use rustyline::highlight::Highlighter; @@ -13,12 +13,12 @@ use rustyline::hint::Hinter; use std::borrow::Cow::{self, Owned}; pub(crate) struct Helper { - helper: ShellManager, + context: Context, } impl Helper { - pub(crate) fn new(helper: ShellManager) -> Helper { - Helper { helper } + pub(crate) fn new(context: Context) -> Helper { + Helper { context } } } @@ -30,7 +30,7 @@ impl Completer for Helper { pos: usize, ctx: &rustyline::Context<'_>, ) -> Result<(usize, Vec), ReadlineError> { - self.helper.complete(line, pos, ctx) + self.context.shell_manager.complete(line, pos, ctx) } } @@ -53,7 +53,7 @@ impl Completer for Helper { impl Hinter for Helper { fn hint(&self, line: &str, pos: usize, ctx: &rustyline::Context<'_>) -> Option { - self.helper.hint(line, pos, ctx) + self.context.shell_manager.hint(line, pos, ctx) } } @@ -78,20 +78,42 @@ impl Highlighter for Helper { Ok(v) => v, }; - let Pipeline { parts } = pipeline; - let mut iter = parts.into_iter(); + let tokens = vec![TokenNode::Pipeline(pipeline.clone().tagged(v.tag()))]; + let mut tokens = TokensIterator::all(&tokens[..], v.tag()); - loop { - match iter.next() { - None => { - return Cow::Owned(out); - } - Some(token) => { - let styled = paint_pipeline_element(&token, line); - out.push_str(&styled.to_string()); - } - } + let text = Text::from(line); + let expand_context = self + .context + .expand_context(&text, Tag::from((0, line.len() - 1, uuid::Uuid::nil()))); + let mut shapes = vec![]; + + // We just constructed a token list that only contains a pipeline, so it can't fail + color_fallible_syntax(&PipelineShape, &mut tokens, &expand_context, &mut shapes) + .unwrap(); + + trace!(target: "nu::shapes", + "SHAPES :: {:?}", + shapes.iter().map(|shape| shape.item).collect::>() + ); + + for shape in shapes { + let styled = paint_flat_shape(shape, line); + out.push_str(&styled); } + + Cow::Owned(out) + + // loop { + // match iter.next() { + // None => { + // return Cow::Owned(out); + // } + // Some(token) => { + // let styled = paint_pipeline_element(&token, line); + // out.push_str(&styled.to_string()); + // } + // } + // } } } } @@ -101,80 +123,55 @@ impl Highlighter for Helper { } } -fn paint_token_node(token_node: &TokenNode, line: &str) -> String { - let styled = match token_node { - TokenNode::Call(..) => Color::Cyan.bold().paint(token_node.tag().slice(line)), - TokenNode::Nodes(..) => Color::Green.bold().paint(token_node.tag().slice(line)), - TokenNode::Whitespace(..) => Color::White.normal().paint(token_node.tag().slice(line)), - TokenNode::Flag(..) => Color::Black.bold().paint(token_node.tag().slice(line)), - TokenNode::Member(..) => Color::Yellow.bold().paint(token_node.tag().slice(line)), - TokenNode::Error(..) => Color::Red.bold().paint(token_node.tag().slice(line)), - TokenNode::Delimited(..) => Color::White.paint(token_node.tag().slice(line)), - TokenNode::Pipeline(..) => Color::Blue.normal().paint(token_node.tag().slice(line)), - TokenNode::Token(Tagged { - item: RawToken::Number(..), - .. - }) => Color::Purple.bold().paint(token_node.tag().slice(line)), - TokenNode::Token(Tagged { - item: RawToken::Size(..), - .. - }) => Color::Purple.bold().paint(token_node.tag().slice(line)), - TokenNode::Token(Tagged { - item: RawToken::GlobPattern, - .. - }) => Color::Cyan.normal().paint(token_node.tag().slice(line)), - TokenNode::Token(Tagged { - item: RawToken::String(..), - .. - }) => Color::Green.normal().paint(token_node.tag().slice(line)), - TokenNode::Token(Tagged { - item: RawToken::Variable(..), - .. - }) => Color::Yellow.bold().paint(token_node.tag().slice(line)), - TokenNode::Token(Tagged { - item: RawToken::Bare, - .. - }) => Color::Green.normal().paint(token_node.tag().slice(line)), - TokenNode::Token(Tagged { - item: RawToken::ExternalCommand(..), - .. - }) => Color::Cyan.bold().paint(token_node.tag().slice(line)), - TokenNode::Token(Tagged { - item: RawToken::ExternalWord, - .. - }) => Color::Black.bold().paint(token_node.tag().slice(line)), - TokenNode::Token(Tagged { - item: RawToken::Operator(..), - .. - }) => Color::Black.bold().paint(token_node.tag().slice(line)), - }; +#[allow(unused)] +fn vec_tag(input: Vec>) -> Option { + let mut iter = input.iter(); + let first = iter.next()?.tag; + let last = iter.last(); - styled.to_string() + Some(match last { + None => first, + Some(last) => first.until(last.tag), + }) } -fn paint_pipeline_element(pipeline_element: &PipelineElement, line: &str) -> String { - let mut styled = String::new(); - - if let Some(_) = pipeline_element.pipe { - styled.push_str(&Color::Purple.paint("|")); - } - - let mut tokens = - TokensIterator::new(&pipeline_element.tokens, pipeline_element.tokens.tag, false); - let head = tokens.next(); - - match head { - None => return styled, - Some(head) => { - styled.push_str(&Color::Cyan.bold().paint(head.tag().slice(line)).to_string()) +fn paint_flat_shape(flat_shape: Tagged, line: &str) -> String { + let style = match &flat_shape.item { + FlatShape::OpenDelimiter(_) => Color::White.normal(), + FlatShape::CloseDelimiter(_) => Color::White.normal(), + FlatShape::ItVariable => Color::Purple.bold(), + FlatShape::Variable => Color::Purple.normal(), + FlatShape::Operator => Color::Yellow.normal(), + FlatShape::Dot => Color::White.normal(), + FlatShape::InternalCommand => Color::Cyan.bold(), + FlatShape::ExternalCommand => Color::Cyan.normal(), + FlatShape::ExternalWord => Color::Black.bold(), + FlatShape::BareMember => Color::Yellow.bold(), + FlatShape::StringMember => Color::Yellow.bold(), + FlatShape::String => Color::Green.normal(), + FlatShape::Path => Color::Cyan.normal(), + FlatShape::GlobPattern => Color::Cyan.bold(), + FlatShape::Word => Color::Green.normal(), + FlatShape::Pipe => Color::Purple.bold(), + FlatShape::Flag => Color::Black.bold(), + FlatShape::ShorthandFlag => Color::Black.bold(), + FlatShape::Int => Color::Purple.bold(), + FlatShape::Decimal => Color::Purple.bold(), + FlatShape::Whitespace => Color::White.normal(), + FlatShape::Error => Color::Red.bold(), + FlatShape::Size { number, unit } => { + let number = number.slice(line); + let unit = unit.slice(line); + return format!( + "{}{}", + Color::Purple.bold().paint(number), + Color::Cyan.bold().paint(unit) + ); } - } + }; - for token in tokens { - styled.push_str(&paint_token_node(token, line)); - } - - styled.to_string() + let body = flat_shape.tag.slice(line); + style.paint(body).to_string() } impl rustyline::Helper for Helper {}