Things work

.cargo/config

@@ -0,0 +1 @@
+paths = ["C:\\Users\\wycat\\Code\\nom_locate"]

Cargo.toml

@@ -19,7 +19,7 @@ prettytable-rs = "0.8.0"
 itertools = "0.8.0"
 ansi_term = "0.11.0"
 conch-parser = "0.1.1"
-nom = "4.2.3"
+nom = "5.0.0-beta2"
 dunce = "1.0.0"
 indexmap = { version = "1.0.2", features = ["serde-1"] }
 chrono-humanize = "0.0.11"
@@ -61,11 +61,10 @@ clipboard = "0.5"
 reqwest = "0.9"
 roxmltree = "0.6.0"
 pretty = "0.5.2"
-nom_locate = "0.3.1"
+nom_locate = { git = "https://github.com/wycats/nom_locate.git", branch = "nom5" }
 derive_more = "0.15.0"
 enum-utils = "0.1.0"
 unicode-xid = "0.1.0"
-nom-trace = { version = "0.1.0", git = "https://github.com/pythondude325/nom-trace.git" }
 
 [dependencies.pancurses]
 version = "0.16"

src/cli.rs

@@ -11,13 +11,13 @@ use crate::commands::classified::{
 use crate::context::Context;
 crate use crate::errors::ShellError;
 use crate::evaluate::Scope;
+use crate::parser::parse2::{PipelineElement, TokenNode};
+use crate::parser::registry;
 
 use crate::git::current_branch;
 use crate::object::Value;
-use crate::parser::ast::{Expression, Leaf, RawExpression};
-use crate::parser::{Args, Pipeline};
 
-use log::debug;
+use log::{debug, trace};
 use rustyline::error::ReadlineError;
 use rustyline::{self, ColorMode, Config, Editor};
 
@@ -59,20 +59,21 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             command("from-xml", from_xml::from_xml),
             command("from-yaml", from_yaml::from_yaml),
             command("get", get::get),
-            command("open", open::open),
             command("enter", enter::enter),
             command("exit", exit::exit),
             command("pick", pick::pick),
             command("split-column", split_column::split_column),
             command("split-row", split_row::split_row),
+            command("lines", lines::lines),
             command("reject", reject::reject),
             command("trim", trim::trim),
             command("to-array", to_array::to_array),
             command("to-json", to_json::to_json),
             command("to-toml", to_toml::to_toml),
+            command("sort-by", sort_by::sort_by),
+            Arc::new(Open),
             Arc::new(Where),
             Arc::new(Config),
-            command("sort-by", sort_by::sort_by),
         ]);
 
         context.add_sinks(vec![
@@ -138,7 +139,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
                     let host = context.host.lock().unwrap();
                     let writer = host.err_termcolor();
                     line.push_str(" ");
-                    let files = crate::parser::span::Files::new(line);
+                    let files = crate::parser::Files::new(line);
 
                     language_reporting::emit(
                         &mut writer.lock(),
@@ -231,7 +232,7 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
             debug!("=== Parsed ===");
             debug!("{:#?}", result);
 
-            let mut pipeline = classify_pipeline(&result, ctx)?;
+            let mut pipeline = classify_pipeline(&result, ctx, &line)?;
 
             match pipeline.commands.last() {
                 Some(ClassifiedCommand::Sink(_)) => {}
@@ -239,9 +240,9 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
                 _ => pipeline.commands.push(ClassifiedCommand::Sink(SinkCommand {
                     command: sink("autoview", autoview::autoview),
                     name_span: None,
-                    args: Args {
+                    args: registry::Args {
-                        positional: vec![],
+                        positional: None,
-                        named: indexmap::IndexMap::new(),
+                        named: None,
                     },
                 })),
             }
@@ -347,14 +348,15 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
 }
 
 fn classify_pipeline(
-    pipeline: &Pipeline,
+    pipeline: &TokenNode,
     context: &Context,
+    source: &str,
 ) -> Result<ClassifiedPipeline, ShellError> {
-    let commands: Result<Vec<_>, _> = pipeline
+    let pipeline = pipeline.as_pipeline()?;
-        .commands
+
+    let commands: Result<Vec<_>, ShellError> = pipeline
         .iter()
-        .cloned()
+        .map(|item| classify_command(&item, context, source))
-        .map(|item| classify_command(&item, context))
         .collect();
 
     Ok(ClassifiedPipeline {
@@ -363,61 +365,50 @@ fn classify_pipeline(
 }
 
 fn classify_command(
-    command: &Expression,
+    command: &PipelineElement,
     context: &Context,
+    source: &str,
 ) -> Result<ClassifiedCommand, ShellError> {
-    // let command_name = &command.name[..];
+    let call = command.call();
-    // let args = &command.args;
 
-    if let Expression {
+    match call {
-        expr: RawExpression::Call(call),
+        call if call.head().is_bare() => {
-        ..
+            let head = call.head();
-    } = command
+            let name = head.source(source);
-    {
+
-        match (&call.name, &call.args) {
+            match context.has_command(name) {
-            (
-                Expression {
-                    expr: RawExpression::Leaf(Leaf::Bare(name)),
-                    span,
-                },
-                args,
-            ) => match context.has_command(&name.to_string()) {
                 true => {
-                    let command = context.get_command(&name.to_string());
+                    let command = context.get_command(name);
                     let config = command.config();
                     let scope = Scope::empty();
 
-                    let args = match args {
+                    trace!("classifying {:?}", config);
-                        Some(args) => config.evaluate_args(args.iter(), &scope)?,
+
-                        None => Args::default(),
+                    let args = config.evaluate_args(call, context, &scope, source)?;
-                    };
 
                     Ok(ClassifiedCommand::Internal(InternalCommand {
                         command,
-                        name_span: Some(span.clone()),
+                        name_span: Some(head.span().clone()),
                         args,
                     }))
                 }
-                false => match context.has_sink(&name.to_string()) {
+                false => match context.has_sink(name) {
                     true => {
-                        let command = context.get_sink(&name.to_string());
+                        let command = context.get_sink(name);
                         let config = command.config();
                         let scope = Scope::empty();
 
-                        let args = match args {
+                        let args = config.evaluate_args(call, context, &scope, source)?;
-                            Some(args) => config.evaluate_args(args.iter(), &scope)?,
-                            None => Args::default(),
-                        };
 
                         Ok(ClassifiedCommand::Sink(SinkCommand {
                             command,
-                            name_span: Some(span.clone()),
+                            name_span: Some(head.span().clone()),
                             args,
                         }))
                     }
                     false => {
-                        let arg_list_strings: Vec<String> = match args {
+                        let arg_list_strings: Vec<String> = match call.children() {
-                            Some(args) => args.iter().map(|i| i.as_external_arg()).collect(),
+                            Some(args) => args.iter().map(|i| i.as_external_arg(source)).collect(),
                             None => vec![],
                         };
 
@@ -427,17 +418,11 @@ fn classify_command(
                         }))
                     }
                 },
-            },
+            }
-
-            (_, None) => Err(ShellError::string(
-                "Unimplemented command that is just an expression (1)",
-            )),
-            (_, Some(_)) => Err(ShellError::string("Unimplemented dynamic command")),
         }
-    } else {
+
-        Err(ShellError::string(&format!(
+        _ => Err(ShellError::unimplemented(
-            "Unimplemented command that is just an expression (2) -- {:?}",
+            "classify_command on command whose head is not bare",
-            command
+        )),
-        )))
     }
 }

src/commands.rs

@@ -13,6 +13,7 @@ crate mod from_toml;
 crate mod from_xml;
 crate mod from_yaml;
 crate mod get;
+crate mod lines;
 crate mod ls;
 crate mod open;
 crate mod pick;
@@ -34,5 +35,5 @@ crate mod where_;
 
 crate use command::command;
 crate use config::Config;
-
+crate use open::Open;
 crate use where_::Where;

src/commands/cd.rs

@@ -10,7 +10,8 @@ pub fn cd(args: CommandArgs) -> Result<OutputStream, ShellError> {
     match latest.obj {
         Value::Filesystem => {
             let cwd = latest.path().to_path_buf();
-            let path = match args.positional.first() {
+
+            let path = match args.nth(0) {
                 None => match dirs::home_dir() {
                     Some(o) => o,
                     _ => return Err(ShellError::string("Can not change to home directory")),
@@ -23,7 +24,7 @@ pub fn cd(args: CommandArgs) -> Result<OutputStream, ShellError> {
                             return Err(ShellError::labeled_error(
                                 "Can not change to directory",
                                 "directory not found",
-                                args.positional[0].span.clone(),
+                                v.span.clone(),
                             ));
                         }
                     }
@@ -34,11 +35,11 @@ pub fn cd(args: CommandArgs) -> Result<OutputStream, ShellError> {
             match env::set_current_dir(&path) {
                 Ok(_) => {}
                 Err(_) => {
-                    if args.positional.len() > 0 {
+                    if args.len() > 0 {
                         return Err(ShellError::labeled_error(
                             "Can not change to directory",
                             "directory not found",
-                            args.positional[0].span.clone(),
+                            args.nth(0).unwrap().span.clone(),
                         ));
                     } else {
                         return Err(ShellError::string("Can not change to directory"));
@@ -50,7 +51,7 @@ pub fn cd(args: CommandArgs) -> Result<OutputStream, ShellError> {
         }
         _ => {
             let mut stream = VecDeque::new();
-            match args.positional.first() {
+            match args.nth(0) {
                 None => {
                     stream.push_back(ReturnValue::change_cwd(PathBuf::from("/")));
                 }

src/commands/classified.rs

@@ -1,14 +1,14 @@
 use crate::commands::command::Sink;
-use crate::parser::ast::Expression;
+use crate::parser::{registry::Args, Span, TokenNode};
-use crate::parser::lexer::Span;
-use crate::parser::registry::Args;
 use crate::prelude::*;
 use bytes::{BufMut, BytesMut};
 use futures_codec::{Decoder, Encoder, Framed};
+use log::{log_enabled, trace};
 use std::io::{Error, ErrorKind};
 use std::path::PathBuf;
 use std::sync::Arc;
 use subprocess::Exec;
+
 /// A simple `Codec` implementation that splits up data into lines.
 pub struct LinesCodec {}
 
@@ -79,7 +79,7 @@ crate struct ClassifiedPipeline {
 
 crate enum ClassifiedCommand {
     #[allow(unused)]
-    Expr(Expression),
+    Expr(TokenNode),
     Internal(InternalCommand),
     Sink(SinkCommand),
     External(ExternalCommand),
@@ -109,12 +109,22 @@ impl InternalCommand {
         context: &mut Context,
         input: ClassifiedInputStream,
     ) -> Result<InputStream, ShellError> {
-        let result = context.run_command(
+        let objects = if log_enabled!(log::Level::Trace) {
-            self.command,
+            trace!("->");
-            self.name_span.clone(),
+            trace!("{}", self.command.name());
-            self.args,
+            trace!("{:?}", self.args.debug());
-            input.objects,
+            let objects: Vec<_> = input.objects.collect().await;
-        )?;
+            trace!(
+                "input = {:#?}",
+                objects.iter().map(|o| o.debug()).collect::<Vec<_>>(),
+            );
+            VecDeque::from(objects).boxed()
+        } else {
+            input.objects
+        };
+
+        let result =
+            context.run_command(self.command, self.name_span.clone(), self.args, objects)?;
         let env = context.env.clone();
 
         let stream = result.filter_map(move |v| match v {
@@ -171,9 +181,10 @@ impl ExternalCommand {
     ) -> Result<ClassifiedInputStream, ShellError> {
         let inputs: Vec<Value> = input.objects.collect().await;
 
+        trace!("{:?} -> {}", inputs, self.name);
+
         let mut arg_string = format!("{}", self.name);
         for arg in &self.args {
-            arg_string.push_str(" ");
             arg_string.push_str(&arg);
         }
 
@@ -184,6 +195,7 @@ impl ExternalCommand {
 
             if arg_string.contains("$it") {
                 let mut first = true;
+
                 for i in &inputs {
                     if !first {
                         process = process.arg("&&");
@@ -193,6 +205,10 @@ impl ExternalCommand {
                     }
 
                     for arg in &self.args {
+                        if arg.chars().all(|c| c.is_whitespace()) {
+                            continue;
+                        }
+
                         process = process.arg(&arg.replace("$it", &i.as_string().unwrap()));
                     }
                 }
@@ -217,6 +233,10 @@ impl ExternalCommand {
                     }
 
                     for arg in &self.args {
+                        if arg.chars().all(|c| c.is_whitespace()) {
+                            continue;
+                        }
+
                         new_arg_string.push_str(" ");
                         new_arg_string.push_str(&arg.replace("$it", &i.as_string().unwrap()));
                     }

src/commands/command.rs

@@ -1,25 +1,53 @@
 use crate::errors::ShellError;
 use crate::object::Value;
-use crate::parser::lexer::Span;
+use crate::parser::{
-use crate::parser::lexer::Spanned;
+    registry::{self, Args},
-use crate::parser::CommandConfig;
+    Span, Spanned,
+};
 use crate::prelude::*;
+use getset::Getters;
 use std::path::PathBuf;
 
+#[derive(Getters)]
+#[get = "crate"]
 pub struct CommandArgs {
     pub host: Arc<Mutex<dyn Host + Send>>,
     pub env: Arc<Mutex<Vec<Environment>>>,
     pub name_span: Option<Span>,
-    pub positional: Vec<Spanned<Value>>,
+    pub args: Args,
-    pub named: indexmap::IndexMap<String, Value>,
     pub input: InputStream,
 }
 
+impl CommandArgs {
+    pub fn nth(&self, pos: usize) -> Option<&Spanned<Value>> {
+        self.args.nth(pos)
+    }
+
+    pub fn positional_iter(&self) -> impl Iterator<Item = &Spanned<Value>> {
+        self.args.positional_iter()
+    }
+
+    pub fn expect_nth(&self, pos: usize) -> Result<&Spanned<Value>, ShellError> {
+        self.args.expect_nth(pos)
+    }
+
+    pub fn len(&self) -> usize {
+        self.args.len()
+    }
+
+    pub fn get(&self, name: &str) -> Option<&Spanned<Value>> {
+        self.args.get(name)
+    }
+
+    pub fn has(&self, name: &str) -> bool {
+        self.args.has(name)
+    }
+}
+
 pub struct SinkCommandArgs {
     pub ctx: Context,
     pub name_span: Option<Span>,
-    pub positional: Vec<Spanned<Value>>,
+    pub args: Args,
-    pub named: indexmap::IndexMap<String, Value>,
     pub input: Vec<Value>,
 }
 
@@ -46,8 +74,8 @@ pub trait Command {
     fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError>;
     fn name(&self) -> &str;
 
-    fn config(&self) -> CommandConfig {
+    fn config(&self) -> registry::CommandConfig {
-        CommandConfig {
+        registry::CommandConfig {
             name: self.name().to_string(),
             mandatory_positional: vec![],
             optional_positional: vec![],
@@ -61,8 +89,8 @@ pub trait Sink {
     fn run(&self, args: SinkCommandArgs) -> Result<(), ShellError>;
     fn name(&self) -> &str;
 
-    fn config(&self) -> CommandConfig {
+    fn config(&self) -> registry::CommandConfig {
-        CommandConfig {
+        registry::CommandConfig {
             name: self.name().to_string(),
             mandatory_positional: vec![],
             optional_positional: vec![],

src/commands/config.rs

@@ -1,8 +1,7 @@
 use crate::errors::ShellError;
 use crate::object::config;
 use crate::object::Value;
-use crate::parser::registry::{NamedType, NamedValue};
+use crate::parser::registry::{CommandConfig, NamedType, NamedValue};
-use crate::parser::CommandConfig;
 use crate::prelude::*;
 use indexmap::IndexMap;
 use log::trace;
@@ -19,7 +18,7 @@ impl Command for Config {
 
     fn config(&self) -> CommandConfig {
         let mut named: IndexMap<String, NamedType> = IndexMap::new();
-        named.insert("set".to_string(), NamedType::Optional(NamedValue::Tuple));
+        named.insert("set".to_string(), NamedType::Optional(NamedValue::Single));
         named.insert("get".to_string(), NamedType::Optional(NamedValue::Single));
         named.insert("clear".to_string(), NamedType::Switch);
 
@@ -41,10 +40,10 @@ impl Command for Config {
 pub fn config(args: CommandArgs) -> Result<OutputStream, ShellError> {
     let mut result = crate::object::config::config()?;
 
-    trace!("{:#?}", args.positional);
+    trace!("{:#?}", args.args.positional);
-    trace!("{:#?}", args.named);
+    trace!("{:#?}", args.args.named);
 
-    if let Some(v) = args.named.get("get") {
+    if let Some(v) = args.get("get") {
         let key = v.as_string()?;
         let value = result
             .get(&key)
@@ -56,7 +55,7 @@ pub fn config(args: CommandArgs) -> Result<OutputStream, ShellError> {
         );
     }
 
-    if let Some(v) = args.named.get("set") {
+    if let Some(v) = args.get("set") {
         if let Ok((key, value)) = v.as_pair() {
             result.insert(key.as_string()?, value.clone());
 
@@ -71,7 +70,7 @@ pub fn config(args: CommandArgs) -> Result<OutputStream, ShellError> {
         }
     }
 
-    if let Some(_) = args.named.get("clear") {
+    if let Some(_) = args.get("clear") {
         result.clear();
 
         config::write_config(&result)?;
@@ -84,7 +83,7 @@ pub fn config(args: CommandArgs) -> Result<OutputStream, ShellError> {
         );
     }
 
-    if let Some(v) = args.named.get("remove") {
+    if let Some(v) = args.get("remove") {
         let key = v.as_string()?;
 
         if result.contains_key(&key) {
@@ -104,7 +103,7 @@ pub fn config(args: CommandArgs) -> Result<OutputStream, ShellError> {
         );
     }
 
-    if args.positional.len() == 0 {
+    if args.len() == 0 {
         return Ok(
             futures::stream::once(futures::future::ready(ReturnValue::Value(Value::Object(
                 result.into(),

src/commands/enter.rs

@@ -1,26 +1,28 @@
 use crate::commands::command::CommandAction;
 use crate::errors::ShellError;
 use crate::object::{Primitive, Value};
-use crate::parser::lexer::Spanned;
+use crate::parser::Spanned;
 use crate::prelude::*;
 use std::path::{Path, PathBuf};
 
 pub fn enter(args: CommandArgs) -> Result<OutputStream, ShellError> {
-    if args.positional.len() == 0 {
+    let path = match args.nth(0) {
-        return Err(ShellError::string("open requires a filepath or url"));
+        None => return Err(ShellError::string("open requires a filepath or url")),
-    }
+        Some(p) => p,
+    };
 
     let cwd = args
-        .env
+        .env()
         .lock()
         .unwrap()
         .first()
         .unwrap()
         .path()
         .to_path_buf();
+
     let mut full_path = PathBuf::from(cwd);
 
-    let (file_extension, contents) = match &args.positional[0].item {
+    let (file_extension, contents) = match path.item() {
         Value::Primitive(Primitive::String(s)) => {
             if s.starts_with("http:") || s.starts_with("https:") {
                 let response = reqwest::get(s);
@@ -43,7 +45,7 @@ pub fn enter(args: CommandArgs) -> Result<OutputStream, ShellError> {
                             return Err(ShellError::labeled_error(
                                 "Web page contents corrupt",
                                 "received garbled data",
-                                args.positional[0].span,
+                                args.expect_nth(0)?.span,
                             ));
                         }
                     },
@@ -51,7 +53,7 @@ pub fn enter(args: CommandArgs) -> Result<OutputStream, ShellError> {
                         return Err(ShellError::labeled_error(
                             "URL could not be opened",
                             "url not found",
-                            args.positional[0].span,
+                            args.expect_nth(0)?.span,
                         ));
                     }
                 }
@@ -68,7 +70,7 @@ pub fn enter(args: CommandArgs) -> Result<OutputStream, ShellError> {
                         return Err(ShellError::labeled_error(
                             "File cound not be opened",
                             "file not found",
-                            args.positional[0].span,
+                            args.expect_nth(0)?.span,
                         ));
                     }
                 }
@@ -78,14 +80,14 @@ pub fn enter(args: CommandArgs) -> Result<OutputStream, ShellError> {
             return Err(ShellError::labeled_error(
                 "Expected string value for filename",
                 "expected filename",
-                args.positional[0].span,
+                args.expect_nth(0)?.span,
             ));
         }
     };
 
     let mut stream = VecDeque::new();
 
-    let open_raw = match args.positional.get(1) {
+    let open_raw = match args.nth(1) {
         Some(Spanned {
             item: Value::Primitive(Primitive::String(s)),
             ..

src/commands/exit.rs

@@ -1,11 +1,8 @@
 use crate::commands::command::CommandAction;
 use crate::errors::ShellError;
-use crate::object::{Primitive, Value};
-use crate::parser::lexer::Spanned;
 use crate::prelude::*;
-use std::path::{Path, PathBuf};
 
-pub fn exit(args: CommandArgs) -> Result<OutputStream, ShellError> {
+pub fn exit(_args: CommandArgs) -> Result<OutputStream, ShellError> {
     let mut stream = VecDeque::new();
     stream.push_back(ReturnValue::Action(CommandAction::Exit));
     Ok(stream.boxed())

src/commands/first.rs

@@ -4,7 +4,7 @@ use crate::prelude::*;
 // TODO: "Amount remaining" wrapper
 
 pub fn first(args: CommandArgs) -> Result<OutputStream, ShellError> {
-    if args.positional.len() == 0 {
+    if args.len() == 0 {
         if let Some(span) = args.name_span {
             return Err(ShellError::labeled_error(
                 "First requires an amount",
@@ -16,7 +16,7 @@ pub fn first(args: CommandArgs) -> Result<OutputStream, ShellError> {
         }
     }
 
-    let amount = args.positional[0].as_i64();
+    let amount = args.expect_nth(0)?.as_i64();
 
     let amount = match amount {
         Ok(o) => o,
@@ -24,7 +24,7 @@ pub fn first(args: CommandArgs) -> Result<OutputStream, ShellError> {
             return Err(ShellError::labeled_error(
                 "Value is not a number",
                 "expected integer",
-                args.positional[0].span,
+                args.expect_nth(0)?.span,
             ))
         }
     };

src/commands/get.rs

@@ -20,7 +20,7 @@ fn get_member(path: &str, obj: &Value) -> Option<Value> {
 }
 
 pub fn get(args: CommandArgs) -> Result<OutputStream, ShellError> {
-    if args.positional.len() == 0 {
+    if args.len() == 0 {
         if let Some(span) = args.name_span {
             return Err(ShellError::labeled_error(
                 "Get requires a field or field path",
@@ -32,7 +32,7 @@ pub fn get(args: CommandArgs) -> Result<OutputStream, ShellError> {
         }
     }
 
-    let amount = args.positional[0].as_i64();
+    let amount = args.expect_nth(0)?.as_i64();
 
     // If it's a number, get the row instead of the column
     if let Ok(amount) = amount {
@@ -44,7 +44,13 @@ pub fn get(args: CommandArgs) -> Result<OutputStream, ShellError> {
             .boxed());
     }
 
-    let fields: Result<Vec<String>, _> = args.positional.iter().map(|a| a.as_string()).collect();
+    let fields: Result<Vec<String>, _> = args
+        .args
+        .positional
+        .unwrap()
+        .iter()
+        .map(|a| a.as_string())
+        .collect();
     let fields = fields?;
 
     let stream = args

src/commands/lines.rs

@@ -0,0 +1,35 @@
+use crate::errors::ShellError;
+use crate::object::{Primitive, Value};
+use crate::prelude::*;
+use log::trace;
+
+// TODO: "Amount remaining" wrapper
+
+pub fn lines(args: CommandArgs) -> Result<OutputStream, ShellError> {
+    let input = args.input;
+
+    let stream = input
+        .map(move |v| match v {
+            Value::Primitive(Primitive::String(s)) => {
+                let splitter = "\n";
+                let split_result: Vec<_> = s.split(&splitter).filter(|s| s.trim() != "").collect();
+
+                trace!("split result = {:?}", split_result);
+
+                let mut result = VecDeque::new();
+                for s in split_result {
+                    result.push_back(ReturnValue::Value(Value::Primitive(Primitive::String(
+                        s.to_string(),
+                    ))));
+                }
+                result
+            }
+            _ => {
+                let result = VecDeque::new();
+                result
+            }
+        })
+        .flatten();
+
+    Ok(stream.boxed())
+}

src/commands/ls.rs

@@ -1,6 +1,6 @@
 use crate::errors::ShellError;
 use crate::object::{dir_entry_dict, Primitive, Value};
-use crate::parser::lexer::Spanned;
+use crate::parser::Spanned;
 use crate::prelude::*;
 use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
@@ -10,7 +10,7 @@ pub fn ls(args: CommandArgs) -> Result<OutputStream, ShellError> {
     let path = env.last().unwrap().path.to_path_buf();
     let obj = &env.last().unwrap().obj;
     let mut full_path = PathBuf::from(path);
-    match &args.positional.get(0) {
+    match &args.nth(0) {
         Some(Spanned {
             item: Value::Primitive(Primitive::String(s)),
             ..
@@ -24,7 +24,7 @@ pub fn ls(args: CommandArgs) -> Result<OutputStream, ShellError> {
 
             let entries = match entries {
                 Err(e) => {
-                    if let Some(s) = args.positional.get(0) {
+                    if let Some(s) = args.nth(0) {
                         return Err(ShellError::labeled_error(
                             e.to_string(),
                             e.to_string(),

src/commands/open.rs

@@ -1,11 +1,36 @@
 use crate::errors::ShellError;
 use crate::object::{Primitive, Value};
-use crate::parser::lexer::Spanned;
+use crate::parser::registry::{CommandConfig, NamedType};
 use crate::prelude::*;
+use indexmap::IndexMap;
 use std::path::{Path, PathBuf};
 
-pub fn open(args: CommandArgs) -> Result<OutputStream, ShellError> {
+pub struct Open;
-    if args.positional.len() == 0 {
+
+impl Command for Open {
+    fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
+        open(args)
+    }
+    fn name(&self) -> &str {
+        "open"
+    }
+
+    fn config(&self) -> CommandConfig {
+        let mut named: IndexMap<String, NamedType> = IndexMap::new();
+        named.insert("raw".to_string(), NamedType::Switch);
+
+        CommandConfig {
+            name: self.name().to_string(),
+            mandatory_positional: vec![],
+            optional_positional: vec![],
+            rest_positional: false,
+            named,
+        }
+    }
+}
+
+fn open(args: CommandArgs) -> Result<OutputStream, ShellError> {
+    if args.len() == 0 {
         return Err(ShellError::string("open requires a filepath or url"));
     }
 
@@ -19,7 +44,7 @@ pub fn open(args: CommandArgs) -> Result<OutputStream, ShellError> {
         .to_path_buf();
     let mut full_path = PathBuf::from(cwd);
 
-    let (file_extension, contents) = match &args.positional[0].item {
+    let (file_extension, contents) = match &args.expect_nth(0)?.item {
         Value::Primitive(Primitive::String(s)) => {
             if s.starts_with("http:") || s.starts_with("https:") {
                 let response = reqwest::get(s);
@@ -42,7 +67,7 @@ pub fn open(args: CommandArgs) -> Result<OutputStream, ShellError> {
                             return Err(ShellError::labeled_error(
                                 "Web page contents corrupt",
                                 "received garbled data",
-                                args.positional[0].span,
+                                args.expect_nth(0)?.span,
                             ));
                         }
                     },
@@ -50,7 +75,7 @@ pub fn open(args: CommandArgs) -> Result<OutputStream, ShellError> {
                         return Err(ShellError::labeled_error(
                             "URL could not be opened",
                             "url not found",
-                            args.positional[0].span,
+                            args.expect_nth(0)?.span,
                         ));
                     }
                 }
@@ -67,7 +92,7 @@ pub fn open(args: CommandArgs) -> Result<OutputStream, ShellError> {
                         return Err(ShellError::labeled_error(
                             "File cound not be opened",
                             "file not found",
-                            args.positional[0].span,
+                            args.expect_nth(0)?.span,
                         ));
                     }
                 }
@@ -77,27 +102,14 @@ pub fn open(args: CommandArgs) -> Result<OutputStream, ShellError> {
             return Err(ShellError::labeled_error(
                 "Expected string value for filename",
                 "expected filename",
-                args.positional[0].span,
+                args.expect_nth(0)?.span,
             ));
         }
     };
 
     let mut stream = VecDeque::new();
 
-    let open_raw = match args.positional.get(1) {
+    let open_raw = args.has("raw");
-        Some(Spanned {
-            item: Value::Primitive(Primitive::String(s)),
-            ..
-        }) if s == "--raw" => true,
-        Some(v) => {
-            return Err(ShellError::labeled_error(
-                "Unknown flag for open",
-                "unknown flag",
-                v.span,
-            ))
-        }
-        _ => false,
-    };
 
     match file_extension {
         Some(x) if x == "toml" && !open_raw => {

src/commands/pick.rs

@@ -4,7 +4,7 @@ use crate::object::Value;
 use crate::prelude::*;
 
 pub fn pick(args: CommandArgs) -> Result<OutputStream, ShellError> {
-    if args.positional.len() == 0 {
+    if args.len() == 0 {
         if let Some(span) = args.name_span {
             return Err(ShellError::labeled_error(
                 "Pick requires fields",
@@ -16,7 +16,7 @@ pub fn pick(args: CommandArgs) -> Result<OutputStream, ShellError> {
         }
     }
 
-    let fields: Result<Vec<String>, _> = args.positional.iter().map(|a| a.as_string()).collect();
+    let fields: Result<Vec<String>, _> = args.positional_iter().map(|a| a.as_string()).collect();
     let fields = fields?;
 
     let objects = args

src/commands/reject.rs

@@ -4,7 +4,7 @@ use crate::object::Value;
 use crate::prelude::*;
 
 pub fn reject(args: CommandArgs) -> Result<OutputStream, ShellError> {
-    if args.positional.len() == 0 {
+    if args.len() == 0 {
         if let Some(span) = args.name_span {
             return Err(ShellError::labeled_error(
                 "Reject requires fields",
@@ -16,7 +16,7 @@ pub fn reject(args: CommandArgs) -> Result<OutputStream, ShellError> {
         }
     }
 
-    let fields: Result<Vec<String>, _> = args.positional.iter().map(|a| a.as_string()).collect();
+    let fields: Result<Vec<String>, _> = args.positional_iter().map(|a| a.as_string()).collect();
     let fields = fields?;
 
     let stream = args

src/commands/save.rs

@@ -1,14 +1,19 @@
 use crate::commands::command::SinkCommandArgs;
 use crate::errors::ShellError;
 use crate::object::{Primitive, Value};
-use crate::parser::lexer::Spanned;
+use crate::parser::Spanned;
 use std::path::{Path, PathBuf};
 
 pub fn save(args: SinkCommandArgs) -> Result<(), ShellError> {
-    if args.positional.len() == 0 {
+    if args.args.positional.is_none() {
         return Err(ShellError::string("save requires a filepath"));
     }
 
+    let positional = match args.args.positional {
+        None => return Err(ShellError::string("save requires a filepath")),
+        Some(p) => p,
+    };
+
     let cwd = args
         .ctx
         .env
@@ -19,12 +24,12 @@ pub fn save(args: SinkCommandArgs) -> Result<(), ShellError> {
         .path()
         .to_path_buf();
     let mut full_path = PathBuf::from(cwd);
-    match &(args.positional[0].item) {
+    match &(positional[0].item) {
         Value::Primitive(Primitive::String(s)) => full_path.push(Path::new(s)),
         _ => {}
     }
 
-    let save_raw = match args.positional.get(1) {
+    let save_raw = match positional.get(1) {
         Some(Spanned {
             item: Value::Primitive(Primitive::String(s)),
             ..

src/commands/size.rs

@@ -6,7 +6,7 @@ use std::fs::File;
 use std::io::prelude::*;
 
 pub fn size(args: CommandArgs) -> Result<OutputStream, ShellError> {
-    if args.positional.is_empty() {
+    if args.len() == 0 {
         return Err(ShellError::string("size requires at least one file"));
     }
     let cwd = args
@@ -21,7 +21,7 @@ pub fn size(args: CommandArgs) -> Result<OutputStream, ShellError> {
     let mut contents = String::new();
 
     let mut list = VecDeque::new();
-    for name in args.positional {
+    for name in args.positional_iter() {
         let name = name.as_string()?;
         let path = cwd.join(&name);
         let mut file = File::open(path)?;

src/commands/skip.rs

@@ -2,7 +2,7 @@ use crate::errors::ShellError;
 use crate::prelude::*;
 
 pub fn skip(args: CommandArgs) -> Result<OutputStream, ShellError> {
-    if args.positional.len() == 0 {
+    if args.len() == 0 {
         if let Some(span) = args.name_span {
             return Err(ShellError::labeled_error(
                 "Skip requires an amount",
@@ -14,7 +14,7 @@ pub fn skip(args: CommandArgs) -> Result<OutputStream, ShellError> {
         }
     }
 
-    let amount = args.positional[0].as_i64();
+    let amount = args.expect_nth(0)?.as_i64();
 
     let amount = match amount {
         Ok(o) => o,
@@ -22,7 +22,7 @@ pub fn skip(args: CommandArgs) -> Result<OutputStream, ShellError> {
             return Err(ShellError::labeled_error(
                 "Value is not a number",
                 "expected integer",
-                args.positional[0].span,
+                args.expect_nth(0)?.span,
             ))
         }
     };

src/commands/sort_by.rs

@@ -2,7 +2,7 @@ use crate::errors::ShellError;
 use crate::prelude::*;
 
 pub fn sort_by(args: CommandArgs) -> Result<OutputStream, ShellError> {
-    let fields: Result<Vec<_>, _> = args.positional.iter().map(|a| a.as_string()).collect();
+    let fields: Result<Vec<_>, _> = args.positional_iter().map(|a| a.as_string()).collect();
     let fields = fields?;
 
     let output = args.input.collect::<Vec<_>>();

src/commands/split_column.rs

@@ -6,20 +6,33 @@ use log::trace;
 // TODO: "Amount remaining" wrapper
 
 pub fn split_column(args: CommandArgs) -> Result<OutputStream, ShellError> {
+    let positional: Vec<_> = args.positional_iter().cloned().collect();
+
+    if positional.len() == 0 {
+        if let Some(span) = args.name_span {
+            return Err(ShellError::labeled_error(
+                "split-column requires arguments",
+                "needs parameter",
+                span,
+            ));
+        } else {
+            return Err(ShellError::string("split-column requires arguments."));
+        }
+    }
+
     let input = args.input;
-    let args = args.positional;
 
     Ok(input
         .map(move |v| match v {
             Value::Primitive(Primitive::String(s)) => {
-                let splitter = args[0].as_string().unwrap().replace("\\n", "\n");
+                let splitter = positional[0].as_string().unwrap().replace("\\n", "\n");
                 trace!("splitting with {:?}", splitter);
                 let split_result: Vec<_> = s.split(&splitter).filter(|s| s.trim() != "").collect();
 
                 trace!("split result = {:?}", split_result);
 
                 // If they didn't provide column names, make up our own
-                if (args.len() - 1) == 0 {
+                if (positional.len() - 1) == 0 {
                     let mut gen_columns = vec![];
                     for i in 0..split_result.len() {
                         gen_columns.push(format!("Column{}", i + 1));
@@ -33,9 +46,9 @@ pub fn split_column(args: CommandArgs) -> Result<OutputStream, ShellError> {
                         );
                     }
                     ReturnValue::Value(Value::Object(dict))
-                } else if split_result.len() == (args.len() - 1) {
+                } else if split_result.len() == (positional.len() - 1) {
                     let mut dict = crate::object::Dictionary::default();
-                    for (k, v) in split_result.iter().zip(args.iter().skip(1)) {
+                    for (k, v) in split_result.iter().zip(positional.iter().skip(1)) {
                         dict.add(
                             v.as_string().unwrap(),
                             Value::Primitive(Primitive::String(k.to_string())),
@@ -44,7 +57,7 @@ pub fn split_column(args: CommandArgs) -> Result<OutputStream, ShellError> {
                     ReturnValue::Value(Value::Object(dict))
                 } else {
                     let mut dict = crate::object::Dictionary::default();
-                    for k in args.iter().skip(1) {
+                    for k in positional.iter().skip(1) {
                         dict.add(
                             k.as_string().unwrap().trim(),
                             Value::Primitive(Primitive::String("".to_string())),

src/commands/split_row.rs

@@ -1,18 +1,32 @@
 use crate::errors::ShellError;
 use crate::object::{Primitive, Value};
+use crate::parser::Spanned;
 use crate::prelude::*;
 use log::trace;
 
 // TODO: "Amount remaining" wrapper
 
 pub fn split_row(args: CommandArgs) -> Result<OutputStream, ShellError> {
+    let positional: Vec<Spanned<Value>> = args.positional_iter().cloned().collect();
+
+    if positional.len() == 0 {
+        if let Some(span) = args.name_span {
+            return Err(ShellError::labeled_error(
+                "split-row requires arguments",
+                "needs parameter",
+                span,
+            ));
+        } else {
+            return Err(ShellError::string("split-row requires arguments."));
+        }
+    }
+
     let input = args.input;
-    let args = args.positional;
 
     let stream = input
         .map(move |v| match v {
             Value::Primitive(Primitive::String(s)) => {
-                let splitter = args[0].as_string().unwrap().replace("\\n", "\n");
+                let splitter = positional[0].as_string().unwrap().replace("\\n", "\n");
                 trace!("splitting with {:?}", splitter);
                 let split_result: Vec<_> = s.split(&splitter).filter(|s| s.trim() != "").collect();
 

src/commands/view.rs

@@ -3,7 +3,7 @@ use crate::prelude::*;
 use prettyprint::PrettyPrinter;
 
 pub fn view(args: CommandArgs) -> Result<OutputStream, ShellError> {
-    if args.positional.len() == 0 {
+    if args.len() == 0 {
         if let Some(span) = args.name_span {
             return Err(ShellError::labeled_error(
                 "View requires a filename",
@@ -15,7 +15,7 @@ pub fn view(args: CommandArgs) -> Result<OutputStream, ShellError> {
         }
     }
 
-    let target = match args.positional[0].as_string() {
+    let target = match args.expect_nth(0)?.as_string() {
         Ok(s) => s.clone(),
         Err(e) => {
             if let Some(span) = args.name_span {

src/commands/where_.rs

@@ -1,6 +1,5 @@
 use crate::errors::ShellError;
-use crate::parser::registry::PositionalType;
+use crate::parser::registry::{CommandConfig, PositionalType};
-use crate::parser::CommandConfig;
 use crate::prelude::*;
 
 pub struct Where;
@@ -25,11 +24,11 @@ impl Command for Where {
 }
 
 pub fn r#where(args: CommandArgs) -> Result<OutputStream, ShellError> {
-    if args.positional.is_empty() {
+    if args.len() == 0 {
         return Err(ShellError::string("select requires a field"));
     }
 
-    let block = args.positional[0].as_block()?;
+    let block = args.expect_nth(0)?.as_block()?;
     let input = args.input;
 
     let objects = input.filter_map(move |item| {

src/context.rs

@@ -1,7 +1,8 @@
-use crate::commands::command::Sink;
+use crate::commands::command::{Sink, SinkCommandArgs};
-use crate::commands::command::SinkCommandArgs;
+use crate::parser::{
-use crate::parser::lexer::Span;
+    registry::{Args, CommandConfig, CommandRegistry},
-use crate::parser::Args;
+    Span,
+};
 use crate::prelude::*;
 
 use indexmap::IndexMap;
@@ -56,8 +57,7 @@ impl Context {
         let command_args = SinkCommandArgs {
             ctx: self.clone(),
             name_span,
-            positional: args.positional,
+            args,
-            named: args.named,
             input,
         };
 
@@ -87,11 +87,16 @@ impl Context {
             host: self.host.clone(),
             env: self.env.clone(),
             name_span,
-            positional: args.positional,
+            args,
-            named: args.named,
             input,
         };
 
         command.run(command_args)
     }
 }
+
+impl CommandRegistry for Context {
+    fn get(&self, name: &str) -> Option<CommandConfig> {
+        self.commands.get(name).map(|c| c.config())
+    }
+}

src/errors.rs

@@ -1,6 +1,7 @@
-use crate::parser::lexer::{Span, SpannedToken};
 #[allow(unused)]
 use crate::prelude::*;
+
+use crate::parser::Span;
 use derive_new::new;
 use language_reporting::{Diagnostic, Label, Severity};
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -15,26 +16,40 @@ pub enum ShellError {
 
 impl ShellError {
     crate fn parse_error(
-        error: lalrpop_util::ParseError<usize, SpannedToken, ShellError>,
+        error: nom::Err<(nom_locate::LocatedSpan<&str>, nom::error::ErrorKind)>,
     ) -> ShellError {
-        use lalrpop_util::ParseError;
         use language_reporting::*;
 
         match error {
-            ParseError::UnrecognizedToken {
+            nom::Err::Incomplete(_) => unreachable!(),
-                token: (start, SpannedToken { token, .. }, end),
+            nom::Err::Failure(span) | nom::Err::Error(span) => {
-                expected,
+                let diagnostic =
-            } => {
+                    Diagnostic::new(Severity::Error, format!("{:?}", span))
-                let diagnostic = Diagnostic::new(
+                        .with_label(Label::new_primary(Span::from(span.0)));
-                    Severity::Error,
-                    format!("Unexpected {:?}, expected {:?}", token, expected),
-                )
-                .with_label(Label::new_primary(Span::from((start, end))));
 
                 ShellError::diagnostic(diagnostic)
+                // nom::Context::Code(span, kind) => {
+                //     let diagnostic =
+                //         Diagnostic::new(Severity::Error, format!("{}", kind.description()))
+                //             .with_label(Label::new_primary(Span::from(span)));
+
+                //     ShellError::diagnostic(diagnostic)
+                // }
             }
-            ParseError::User { error } => error,
+            // ParseError::UnrecognizedToken {
-            other => ShellError::string(format!("{:?}", other)),
+            //     token: (start, SpannedToken { token, .. }, end),
+            //     expected,
+            // } => {
+            //     let diagnostic = Diagnostic::new(
+            //         Severity::Error,
+            //         format!("Unexpected {:?}, expected {:?}", token, expected),
+            //     )
+            //     .with_label(Label::new_primary(Span::from((start, end))));
+
+            //     ShellError::diagnostic(diagnostic)
+            // }
+            // ParseError::User { error } => error,
+            // other => ShellError::string(format!("{:?}", other)),
         }
     }
 
@@ -61,6 +76,10 @@ impl ShellError {
         ShellError::string(&format!("Unimplemented: {}", title.into()))
     }
 
+    crate fn unexpected(title: impl Into<String>) -> ShellError {
+        ShellError::string(&format!("Unexpected: {}", title.into()))
+    }
+
     crate fn copy_error(&self) -> ShellError {
         self.clone()
     }
@@ -177,14 +196,14 @@ impl std::convert::From<subprocess::PopenError> for ShellError {
     }
 }
 
-impl std::convert::From<nom::Err<(&str, nom::ErrorKind)>> for ShellError {
+// impl std::convert::From<nom::Err<(&str, nom::ErrorKind)>> for ShellError {
-    fn from(input: nom::Err<(&str, nom::ErrorKind)>) -> ShellError {
+//     fn from(input: nom::Err<(&str, nom::ErrorKind)>) -> ShellError {
-        ShellError::String(StringError {
+//         ShellError::String(StringError {
-            title: format!("{:?}", input),
+//             title: format!("{:?}", input),
-            error: Value::nothing(),
+//             error: Value::nothing(),
-        })
+//         })
-    }
+//     }
-}
+// }
 
 impl std::convert::From<toml::ser::Error> for ShellError {
     fn from(input: toml::ser::Error) -> ShellError {

src/evaluate/evaluator.rs

@@ -1,6 +1,8 @@
-use crate::object::Primitive;
+use crate::object::base::Block;
-use crate::parser::ast;
+use crate::parser::{
-use crate::parser::lexer::Spanned;
+    hir::{self, Expression, RawExpression},
+    CommandRegistry, Span, Spanned, Text,
+};
 use crate::prelude::*;
 use derive_new::new;
 use indexmap::IndexMap;
@@ -21,96 +23,75 @@ impl Scope {
     }
 }
 
-crate fn evaluate_expr(
+crate fn evaluate_baseline_expr(
-    expr: &ast::Expression,
+    expr: &Expression,
+    registry: &dyn CommandRegistry,
     scope: &Scope,
+    source: &str,
 ) -> Result<Spanned<Value>, ShellError> {
-    use ast::*;
+    match &expr.item {
-    match &expr.expr {
+        RawExpression::Literal(literal) => Ok(evaluate_literal(expr.copy_span(*literal), source)),
-        RawExpression::Call(_) => Err(ShellError::unimplemented("Evaluating call expression")),
+        RawExpression::Variable(var) => evaluate_reference(var, scope, source),
-        RawExpression::Leaf(l) => Ok(Spanned::from_item(evaluate_leaf(l), expr.span.clone())),
+        RawExpression::Binary(binary) => {
-        RawExpression::Parenthesized(p) => evaluate_expr(&p.expr, scope),
+            let left = evaluate_baseline_expr(binary.left(), registry, scope, source)?;
-        RawExpression::Flag(f) => Ok(Spanned::from_item(
+            let right = evaluate_baseline_expr(binary.right(), registry, scope, source)?;
-            Value::Primitive(Primitive::String(f.print())),
-            expr.span.clone(),
-        )),
-        RawExpression::Block(b) => evaluate_block(&b, scope),
-        RawExpression::Path(p) => evaluate_path(&p, scope),
-        RawExpression::Binary(b) => evaluate_binary(b, scope),
-        RawExpression::VariableReference(r) => {
-            evaluate_reference(r, scope).map(|x| Spanned::from_item(x, expr.span.clone()))
-        }
-    }
-}
 
-fn evaluate_leaf(leaf: &ast::Leaf) -> Value {
+            match left.compare(binary.op(), &*right) {
-    use ast::*;
+                Some(result) => Ok(Spanned::from_item(Value::boolean(result), *expr.span())),
-
+                None => Err(ShellError::unimplemented(&format!(
-    match leaf {
+                    "Comparison failure {:?}",
-        Leaf::String(s) => Value::string(s),
+                    binary
-        Leaf::Bare(path) => Value::string(path.to_string()),
+                ))),
-        Leaf::Boolean(b) => Value::boolean(*b),
-        Leaf::Int(i) => Value::int(*i),
-        Leaf::Unit(i, unit) => unit.compute(*i),
-    }
-}
-
-fn evaluate_reference(r: &ast::Variable, scope: &Scope) -> Result<Value, ShellError> {
-    use ast::Variable::*;
-
-    match r {
-        It => Ok(scope.it.copy()),
-        Other(s) => Ok(scope
-            .vars
-            .get(s)
-            .map(|v| v.copy())
-            .unwrap_or_else(|| Value::nothing())),
-    }
-}
-
-fn evaluate_binary(binary: &ast::Binary, scope: &Scope) -> Result<Spanned<Value>, ShellError> {
-    let left = evaluate_expr(&binary.left, scope)?;
-    let right = evaluate_expr(&binary.right, scope)?;
-
-    match left.compare(&binary.operator, &right) {
-        Some(v) => Ok(Spanned::from_item(
-            Value::boolean(v),
-            binary.operator.span.clone(),
-        )),
-        None => Err(ShellError::TypeError(format!(
-            "Can't compare {} and {}",
-            left.type_name(),
-            right.type_name()
-        ))),
-    }
-}
-
-fn evaluate_block(block: &ast::Block, _scope: &Scope) -> Result<Spanned<Value>, ShellError> {
-    Ok(Spanned::from_item(
-        Value::block(block.expr.clone()),
-        block.expr.span.clone(),
-    ))
-}
-
-fn evaluate_path(path: &ast::Path, scope: &Scope) -> Result<Spanned<Value>, ShellError> {
-    let head = path.head();
-    let mut value = evaluate_expr(head, scope)?;
-    let mut seen = vec![];
-
-    for name in path.tail() {
-        let next = value.get_data_by_key(&name.item);
-        seen.push(name.item.clone());
-
-        match next {
-            None => {
-                return Err(ShellError::MissingProperty {
-                    expr: path.print(),
-                    subpath: itertools::join(seen, "."),
-                });
             }
-            Some(v) => value = Spanned::from_item(v.copy(), name.span.clone()),
         }
-    }
+        RawExpression::Block(block) => Ok(Spanned::from_item(
+            Value::Block(Block::new(*block.clone(), Text::from(source))), // TODO: Pass Text around
+            block.span(),
+        )),
+        RawExpression::Path(path) => {
+            let value = evaluate_baseline_expr(path.head(), registry, scope, source)?;
+            let mut value = value.item();
 
-    Ok(value)
+            for name in path.tail() {
+                let next = value.get_data_by_key(name);
+
+                match next {
+                    None => return Err(ShellError::unimplemented("Invalid property from path")),
+                    Some(next) => value = next,
+                };
+            }
+
+            Ok(Spanned::from_item(value.clone(), expr.span()))
+        }
+        RawExpression::Boolean(_boolean) => unimplemented!(),
+    }
+}
+
+fn evaluate_literal(literal: Spanned<hir::Literal>, source: &str) -> Spanned<Value> {
+    let result = match literal.item {
+        hir::Literal::Integer(int) => Value::int(int),
+        hir::Literal::Size(_int, _unit) => unimplemented!(),
+        hir::Literal::String(span) => Value::string(span.slice(source)),
+        hir::Literal::Bare => Value::string(literal.span().slice(source)),
+    };
+
+    literal.map(|_| result)
+}
+
+fn evaluate_reference(
+    name: &hir::Variable,
+    scope: &Scope,
+    source: &str,
+) -> Result<Spanned<Value>, ShellError> {
+    match name {
+        hir::Variable::It(span) => Ok(Spanned::from_item(scope.it.copy(), span)),
+        hir::Variable::Other(span) => Ok(Spanned::from_item(
+            scope
+                .vars
+                .get(span.slice(source))
+                .map(|v| v.copy())
+                .unwrap_or_else(|| Value::nothing()),
+            span,
+        )),
+    }
 }

src/evaluate/mod.rs

@@ -1,3 +1,3 @@
 crate mod evaluator;
 
-crate use evaluator::{evaluate_expr, Scope};
+crate use evaluator::{evaluate_baseline_expr, Scope};

src/object/base.rs

@@ -1,16 +1,17 @@
 use crate::errors::ShellError;
-use crate::evaluate::{evaluate_expr, Scope};
+use crate::evaluate::{evaluate_baseline_expr, Scope};
 use crate::object::DataDescriptor;
-use crate::parser::ast::{self, Operator};
+use crate::parser::{hir, Operator, Spanned};
-use crate::parser::lexer::Spanned;
 use crate::prelude::*;
 use ansi_term::Color;
 use chrono::{DateTime, Utc};
 use chrono_humanize::Humanize;
 use derive_new::new;
 use ordered_float::OrderedFloat;
+use std::fmt;
 use std::time::SystemTime;
 
+use crate::parser::Text;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 
 #[derive(Debug, Clone, Copy, Ord, PartialOrd, Eq, PartialEq, new)]
@@ -75,6 +76,20 @@ impl Primitive {
         .to_string()
     }
 
+    crate fn debug(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        use Primitive::*;
+
+        match self {
+            Nothing => write!(f, "Nothing"),
+            Int(int) => write!(f, "{}", int),
+            Float(float) => write!(f, "{:?}", float),
+            Bytes(bytes) => write!(f, "{}", bytes),
+            String(string) => write!(f, "{:?}", string),
+            Boolean(boolean) => write!(f, "{}", boolean),
+            Date(date) => write!(f, "{}", date),
+        }
+    }
+
     crate fn format(&self, field_name: Option<&DataDescriptor>) -> String {
         match self {
             Primitive::Nothing => format!("{}", Color::Black.bold().paint("-")),
@@ -117,7 +132,8 @@ pub struct Operation {
 
 #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new)]
 pub struct Block {
-    crate expression: ast::Expression,
+    crate expression: hir::Expression,
+    crate source: Text,
 }
 
 impl Serialize for Block {
@@ -125,7 +141,7 @@ impl Serialize for Block {
     where
         S: Serializer,
     {
-        serializer.serialize_str(&self.expression.print())
+        serializer.serialize_str(&self.expression.source(self.source.as_ref()))
     }
 }
 
@@ -134,17 +150,19 @@ impl Deserialize<'de> for Block {
     where
         D: Deserializer<'de>,
     {
-        let mut builder = ast::ExpressionBuilder::new();
+        unimplemented!("deserialize block")
-        let expr: ast::Expression = builder.string("Unserializable block");
+        // let s = "\"unimplemented deserialize block\"";
-
+        // Ok(Block::new(
-        Ok(Block::new(expr))
+        //     TokenTreeBuilder::spanned_string((1, s.len() - 1), (0, s.len())),
+        //     Text::from(s),
+        // ))
     }
 }
 
 impl Block {
     pub fn invoke(&self, value: &Value) -> Result<Spanned<Value>, ShellError> {
         let scope = Scope::new(value.copy());
-        evaluate_expr(&self.expression, &scope)
+        evaluate_baseline_expr(&self.expression, &(), &scope, self.source.as_ref())
     }
 }
 
@@ -153,6 +171,7 @@ pub enum Value {
     Primitive(Primitive),
     Object(crate::object::Dictionary),
     List(Vec<Value>),
+    #[allow(unused)]
     Block(Block),
     Filesystem,
 
@@ -160,6 +179,39 @@ pub enum Value {
     Error(Box<ShellError>),
 }
 
+pub fn debug_list(values: &'a Vec<Value>) -> ValuesDebug<'a> {
+    ValuesDebug { values }
+}
+
+pub struct ValuesDebug<'a> {
+    values: &'a Vec<Value>,
+}
+
+impl fmt::Debug for ValuesDebug<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.debug_list()
+            .entries(self.values.iter().map(|i| i.debug()))
+            .finish()
+    }
+}
+
+pub struct ValueDebug<'a> {
+    value: &'a Value,
+}
+
+impl fmt::Debug for ValueDebug<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self.value {
+            Value::Primitive(p) => p.debug(f),
+            Value::Object(o) => o.debug(f),
+            Value::List(l) => debug_list(l).fmt(f),
+            Value::Block(_) => write!(f, "[[block]]"),
+            Value::Error(err) => write!(f, "[[error :: {} ]]", err),
+            Value::Filesystem => write!(f, "[[filesystem]]"),
+        }
+    }
+}
+
 impl Value {
     crate fn type_name(&self) -> String {
         match self {
@@ -172,6 +224,10 @@ impl Value {
         }
     }
 
+    crate fn debug(&'a self) -> ValueDebug<'a> {
+        ValueDebug { value: self }
+    }
+
     crate fn data_descriptors(&self) -> Vec<DataDescriptor> {
         match self {
             Value::Primitive(_) => vec![DataDescriptor::value_of()],
@@ -237,7 +293,7 @@ impl Value {
     crate fn format_leaf(&self, desc: Option<&DataDescriptor>) -> String {
         match self {
             Value::Primitive(p) => p.format(desc),
-            Value::Block(b) => b.expression.print(),
+            Value::Block(b) => b.expression.source(b.source.as_ref()).to_string(),
             Value::Object(_) => format!("[object Object]"),
             Value::List(_) => format!("[list List]"),
             Value::Error(e) => format!("{}", e),
@@ -245,7 +301,8 @@ impl Value {
         }
     }
 
-    crate fn compare(&self, operator: &ast::Operator, other: &Value) -> Option<bool> {
+    #[allow(unused)]
+    crate fn compare(&self, operator: &Operator, other: &Value) -> Option<bool> {
         match operator {
             _ => {
                 let coerced = coerce_compare(self, other)?;
@@ -342,8 +399,9 @@ impl Value {
         }
     }
 
-    crate fn block(e: ast::Expression) -> Value {
+    #[allow(unused)]
-        Value::Block(Block::new(e))
+    crate fn block(e: hir::Expression, source: Text) -> Value {
+        Value::Block(Block::new(e, source))
     }
 
     crate fn string(s: impl Into<String>) -> Value {

src/object/desc.rs

@@ -16,6 +16,13 @@ impl DescriptorName {
         }
     }
 
+    crate fn debug(&self) -> &str {
+        match self {
+            DescriptorName::String(s) => s,
+            DescriptorName::ValueOf => "[[value]]",
+        }
+    }
+
     crate fn as_string(&self) -> Option<&str> {
         match self {
             DescriptorName::String(s) => Some(s),

src/object/dict.rs

@@ -7,6 +7,7 @@ use indexmap::IndexMap;
 use serde::ser::{Serialize, SerializeMap, Serializer};
 use serde_derive::Deserialize;
 use std::cmp::{Ordering, PartialOrd};
+use std::fmt;
 
 #[derive(Debug, Default, Eq, PartialEq, Deserialize, Clone, new)]
 pub struct Dictionary {
@@ -113,4 +114,14 @@ impl Dictionary {
             None => None,
         }
     }
+
+    crate fn debug(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let mut debug = f.debug_struct("Dictionary");
+
+        for (desc, value) in self.entries.iter() {
+            debug.field(desc.name.debug(), &value.debug());
+        }
+
+        debug.finish()
+    }
 }

src/parser.rs

@@ -1,279 +1,275 @@
-crate mod ast;
+crate mod hir;
-crate mod completer;
-crate mod lexer;
-crate mod parser;
-crate mod registry;
-crate mod span;
 crate mod parse2;
-
+crate mod parse_command;
-crate use ast::Pipeline;
+crate mod registry;
-crate use registry::{Args, CommandConfig};
 
 use crate::errors::ShellError;
-use ast::Module;
-use lexer::Lexer;
-use log::trace;
-use parser::{ModuleParser, ReplLineParser};
 
-pub fn parse(input: &str) -> Result<Pipeline, ShellError> {
+crate use hir::baseline_parse_tokens::baseline_parse_tokens;
+crate use parse2::call_node::CallNode;
+crate use parse2::files::Files;
+crate use parse2::flag::Flag;
+crate use parse2::operator::Operator;
+crate use parse2::parser::{nom_input, pipeline};
+crate use parse2::span::{Span, Spanned};
+crate use parse2::text::Text;
+crate use parse2::token_tree::TokenNode;
+crate use parse2::tokens::{RawToken, Token};
+crate use parse2::unit::Unit;
+crate use parse_command::parse_command;
+crate use registry::CommandRegistry;
+
+pub fn parse(input: &str) -> Result<TokenNode, ShellError> {
     let _ = pretty_env_logger::try_init();
 
-    let parser = ReplLineParser::new();
+    match pipeline(nom_input(input)) {
-    let tokens = Lexer::new(input, false);
+        Ok((_rest, val)) => Ok(val),
-
-    trace!(
-        "Tokens: {:?}",
-        tokens.clone().collect::<Result<Vec<_>, _>>()
-    );
-
-    match parser.parse(tokens) {
-        Ok(val) => Ok(val),
         Err(err) => Err(ShellError::parse_error(err)),
     }
 }
 
-#[allow(unused)]
+// #[allow(unused)]
-pub fn parse_module(input: &str) -> Result<Module, ShellError> {
+// pub fn parse_module(input: &str) -> Result<Module, ShellError> {
-    let _ = pretty_env_logger::try_init();
+//     let _ = pretty_env_logger::try_init();
 
-    let parser = ModuleParser::new();
+//     let parser = ModuleParser::new();
-    let tokens = Lexer::new(input, false);
+//     let tokens = Lexer::new(input, false);
 
-    trace!(
+//     trace!(
-        "Tokens: {:?}",
+//         "Tokens: {:?}",
-        tokens.clone().collect::<Result<Vec<_>, _>>()
+//         tokens.clone().collect::<Result<Vec<_>, _>>()
-    );
+//     );
 
-    match parser.parse(tokens) {
+//     match parser.parse(tokens) {
-        Ok(val) => Ok(val),
+//         Ok(val) => Ok(val),
-        Err(err) => Err(ShellError::parse_error(err)),
+//         Err(err) => Err(ShellError::parse_error(err)),
-    }
+//     }
-}
+// }
 
-#[cfg(test)]
+// #[cfg(test)]
-mod tests {
+// mod tests {
-    use super::*;
+//     use super::*;
-    use crate::parser::ast::Pipeline;
+//     use crate::parser::ast::Pipeline;
-    use pretty_assertions::assert_eq;
+//     use pretty_assertions::assert_eq;
 
-    fn assert_parse(source: &str, expected: Pipeline) {
+//     fn assert_parse(source: &str, expected: Pipeline) {
-        let parsed = match parse(source) {
+//         let parsed = match parse(source) {
-            Ok(p) => p,
+//             Ok(p) => p,
-            Err(ShellError::Diagnostic(diag)) => {
+//             Err(ShellError::Diagnostic(diag)) => {
-                use language_reporting::termcolor;
+//                 use language_reporting::termcolor;
 
-                let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto);
+//                 let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto);
-                let files = crate::parser::span::Files::new(source.to_string());
+//                 let files = crate::parser::span::Files::new(source.to_string());
 
-                language_reporting::emit(
+//                 language_reporting::emit(
-                    &mut writer.lock(),
+//                     &mut writer.lock(),
-                    &files,
+//                     &files,
-                    &diag.diagnostic,
+//                     &diag.diagnostic,
-                    &language_reporting::DefaultConfig,
+//                     &language_reporting::DefaultConfig,
-                )
+//                 )
-                .unwrap();
+//                 .unwrap();
 
-                panic!("Test failed")
+//                 panic!("Test failed")
-            }
+//             }
-            Err(err) => panic!("Something went wrong during parse: {:#?}", err),
+//             Err(err) => panic!("Something went wrong during parse: {:#?}", err),
-        };
+//         };
 
-        let printed = parsed.print();
+//         let printed = parsed.print();
 
-        assert_eq!(parsed, expected);
+//         assert_eq!(parsed, expected);
-        assert_eq!(printed, source);
+//         assert_eq!(printed, source);
 
-        let span = expected.span;
+//         let span = expected.span;
 
-        let expected_module = ModuleBuilder::spanned_items(
+//         let expected_module = ModuleBuilder::spanned_items(
-            vec![Spanned::from_item(RawItem::Expression(expected), span)],
+//             vec![Spanned::from_item(RawItem::Expression(expected), span)],
-            span.start,
+//             span.start,
-            span.end,
+//             span.end,
-        );
+//         );
 
-        assert_parse_module(source, expected_module);
+//         assert_parse_module(source, expected_module);
-    }
+//     }
 
-    fn assert_parse_module(source: &str, expected: Module) {
+//     fn assert_parse_module(source: &str, expected: Module) {
-        let parsed = match parse_module(source) {
+//         let parsed = match parse_module(source) {
-            Ok(p) => p,
+//             Ok(p) => p,
-            Err(ShellError::Diagnostic(diag)) => {
+//             Err(ShellError::Diagnostic(diag)) => {
-                use language_reporting::termcolor;
+//                 use language_reporting::termcolor;
 
-                let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto);
+//                 let writer = termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto);
-                let files = crate::parser::span::Files::new(source.to_string());
+//                 let files = crate::parser::span::Files::new(source.to_string());
 
-                language_reporting::emit(
+//                 language_reporting::emit(
-                    &mut writer.lock(),
+//                     &mut writer.lock(),
-                    &files,
+//                     &files,
-                    &diag.diagnostic,
+//                     &diag.diagnostic,
-                    &language_reporting::DefaultConfig,
+//                     &language_reporting::DefaultConfig,
-                )
+//                 )
-                .unwrap();
+//                 .unwrap();
 
-                panic!("Test failed")
+//                 panic!("Test failed")
-            }
+//             }
-            Err(err) => panic!("Something went wrong during parse: {:#?}", err),
+//             Err(err) => panic!("Something went wrong during parse: {:#?}", err),
-        };
+//         };
 
-        let printed = parsed.print();
+//         let printed = parsed.print();
 
-        assert_eq!(parsed, expected);
+//         assert_eq!(parsed, expected);
-        assert_eq!(printed, source);
+//         assert_eq!(printed, source);
-    }
+//     }
 
-    macro_rules! commands {
+//     macro_rules! commands {
-        ( $( ( $name:tt $( $command:ident ( $arg:expr ) )* ) )|* ) => {{
+//         ( $( ( $name:tt $( $command:ident ( $arg:expr ) )* ) )|* ) => {{
-            use $crate::parser::ast::{Expression, ExpressionBuilder};
+//             use $crate::parser::ast::{Expression, ExpressionBuilder};
-            let mut builder = crate::parser::ast::ExpressionBuilder::new();
+//             let mut builder = crate::parser::ast::ExpressionBuilder::new();
 
-            builder.pipeline(vec![
+//             builder.pipeline(vec![
-                $(
+//                 $(
-                    (command!($name $($command($arg))*) as (&dyn Fn(&mut ExpressionBuilder) -> Expression))
+//                     (command!($name $($command($arg))*) as (&dyn Fn(&mut ExpressionBuilder) -> Expression))
-                ),*
+//                 ),*
-            ])
+//             ])
-        }}
+//         }}
-    }
+//     }
 
-    macro_rules! command {
+//     macro_rules! command {
-        ($name:ident) => {
+//         ($name:ident) => {
-            &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call((
+//             &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call((
-                &|b: &mut $crate::parser::ast::ExpressionBuilder| b.bare(stringify!($name)),
+//                 &|b: &mut $crate::parser::ast::ExpressionBuilder| b.bare(stringify!($name)),
-                vec![]
+//                 vec![]
-            ))
+//             ))
-        };
+//         };
 
-        ($name:ident $( $command:ident ( $body:expr ) )*) => {{
+//         ($name:ident $( $command:ident ( $body:expr ) )*) => {{
-            use $crate::parser::ast::{Expression, ExpressionBuilder};
+//             use $crate::parser::ast::{Expression, ExpressionBuilder};
-            &|b: &mut ExpressionBuilder| b.call((
+//             &|b: &mut ExpressionBuilder| b.call((
-                (&|b: &mut ExpressionBuilder| b.bare(stringify!($name))) as (&dyn Fn(&mut ExpressionBuilder) -> Expression),
+//                 (&|b: &mut ExpressionBuilder| b.bare(stringify!($name))) as (&dyn Fn(&mut ExpressionBuilder) -> Expression),
-                vec![$( (&|b: &mut ExpressionBuilder| b.$command($body)) as &dyn Fn(&mut ExpressionBuilder) -> Expression ),* ]))
+//                 vec![$( (&|b: &mut ExpressionBuilder| b.$command($body)) as &dyn Fn(&mut ExpressionBuilder) -> Expression ),* ]))
 
-        }};
+//         }};
 
-        ($name:ident $( $command:ident ( $body:expr ) )*) => {
+//         ($name:ident $( $command:ident ( $body:expr ) )*) => {
-            &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(|b| b.bare(stringify!($name)), vec![ $( |b| b.$command($body) ),* ])
+//             &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(|b| b.bare(stringify!($name)), vec![ $( |b| b.$command($body) ),* ])
-        };
+//         };
 
-        ($name:tt $( $command:ident ( $body:expr ) )*) => {
+//         ($name:tt $( $command:ident ( $body:expr ) )*) => {
-            &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call((&|b| b.bare($name), vec![ $( &|b| b.$command($body) ),* ]))
+//             &|b: &mut $crate::parser::ast::ExpressionBuilder| b.call((&|b| b.bare($name), vec![ $( &|b| b.$command($body) ),* ]))
-        };
+//         };
-    }
+//     }
 
-    #[test]
+//     #[test]
-    fn parse_simple_command() {
+//     fn parse_simple_command() {
-        assert_parse("ls", commands![(ls)]);
+//         assert_parse("ls", commands![(ls)]);
-    }
+//     }
 
-    #[test]
+//     #[test]
-    fn parse_command_with_args() {
+//     fn parse_command_with_args() {
-        assert_parse(
+//         assert_parse(
-            r#"open Cargo.toml | select package.authors | split-row " ""#,
+//             r#"open Cargo.toml | select package.authors | split-row " ""#,
-            commands![
+//             commands![
-                (open bare("Cargo.toml"))
+//                 (open bare("Cargo.toml"))
-                    | (select bare("package.authors"))
+//                     | (select bare("package.authors"))
-                    | ("split-row" string(" "))
+//                     | ("split-row" string(" "))
-            ],
+//             ],
-        );
+//         );
 
-        assert_parse(r#"git add ."#, commands![("git" bare("add") bare("."))]);
+//         assert_parse(r#"git add ."#, commands![("git" bare("add") bare("."))]);
 
-        assert_parse(
+//         assert_parse(
-            "open Cargo.toml | select package.version | echo $it",
+//             "open Cargo.toml | select package.version | echo $it",
-            commands![
+//             commands![
-                (open bare("Cargo.toml"))
+//                 (open bare("Cargo.toml"))
-                    | (select bare("package.version"))
+//                     | (select bare("package.version"))
-                    | (echo var("it"))
+//                     | (echo var("it"))
-            ],
+//             ],
-        );
+//         );
 
-        assert_parse(
+//         assert_parse(
-            "open Cargo.toml --raw",
+//             "open Cargo.toml --raw",
-            commands![(open bare("Cargo.toml") flag("raw"))],
+//             commands![(open bare("Cargo.toml") flag("raw"))],
-        );
+//         );
 
-        assert_parse(
+//         assert_parse(
-            "open Cargo.toml -r",
+//             "open Cargo.toml -r",
-            commands![(open bare("Cargo.toml") shorthand("r"))],
+//             commands![(open bare("Cargo.toml") shorthand("r"))],
-        );
+//         );
 
-        assert_parse(
+//         assert_parse(
-            "open Cargo.toml | from-toml | to-toml",
+//             "open Cargo.toml | from-toml | to-toml",
-            commands![(open bare("Cargo.toml")) | ("from-toml") | ("to-toml")],
+//             commands![(open bare("Cargo.toml")) | ("from-toml") | ("to-toml")],
-        );
+//         );
 
-        assert_parse(
+//         assert_parse(
-            r#"config --get "ignore dups" | format-list"#,
+//             r#"config --get "ignore dups" | format-list"#,
-            commands![(config flag("get") string("ignore dups")) | ("format-list")],
+//             commands![(config flag("get") string("ignore dups")) | ("format-list")],
-        );
+//         );
 
-        assert_parse(
+//         assert_parse(
-            "open Cargo.toml | from-toml | select dependencies | column serde",
+//             "open Cargo.toml | from-toml | select dependencies | column serde",
-            commands![
+//             commands![
-                (open bare("Cargo.toml"))
+//                 (open bare("Cargo.toml"))
-                    | ("from-toml")
+//                     | ("from-toml")
-                    | (select bare("dependencies"))
+//                     | (select bare("dependencies"))
-                    | (column bare("serde"))
+//                     | (column bare("serde"))
-            ],
+//             ],
-        );
+//         );
 
-        assert_parse(
+//         assert_parse(
-            "config --set tabs 2",
+//             "config --set tabs 2",
-            commands![(config flag("set") bare("tabs") int(2))],
+//             commands![(config flag("set") bare("tabs") int(2))],
-        );
+//         );
 
-        assert_parse(
+//         assert_parse(
-            r#"ls | skip 1 | first 2 | select "file name" | rm $it"#,
+//             r#"ls | skip 1 | first 2 | select "file name" | rm $it"#,
-            commands![
+//             commands![
-                (ls)
+//                 (ls)
-                    | (skip int(1))
+//                     | (skip int(1))
-                    | (first int(2))
+//                     | (first int(2))
-                    | (select string("file name"))
+//                     | (select string("file name"))
-                    | (rm var("it"))
+//                     | (rm var("it"))
-            ],
+//             ],
-        );
+//         );
 
-        assert_parse(
+//         assert_parse(
-            r#"git branch --merged | split-row "`n" | where $it != "* master""#,
+//             r#"git branch --merged | split-row "`n" | where $it != "* master""#,
-            commands![
+//             commands![
-                // TODO: Handle escapes correctly. Should we do ` escape because of paths?
+//                 // TODO: Handle escapes correctly. Should we do ` escape because of paths?
-                (git bare("branch") flag("merged")) | ("split-row" string("`n")) | (where binary((&|b| b.var("it"), &|b| b.op("!="), &|b| b.string("* master"))))
+//                 (git bare("branch") flag("merged")) | ("split-row" string("`n")) | (where binary((&|b| b.var("it"), &|b| b.op("!="), &|b| b.string("* master"))))
-            ],
+//             ],
-        );
+//         );
 
-        assert_parse(
+//         assert_parse(
-            r#"open input2.json | from-json | select glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso | where $it > "GML""#,
+//             r#"open input2.json | from-json | select glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso | where $it > "GML""#,
-            commands![
+//             commands![
-                (open bare("input2.json"))
+//                 (open bare("input2.json"))
-                    | ("from-json")
+//                     | ("from-json")
-                    | (select bare("glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso"))
+//                     | (select bare("glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso"))
-                    | (where binary((&|b| b.var("it"), &|b| b.op(">"), &|b| b.string("GML"))))
+//                     | (where binary((&|b| b.var("it"), &|b| b.op(">"), &|b| b.string("GML"))))
-            ]
+//             ]
-        );
+//         );
 
-        assert_parse(
+//         assert_parse(
-            r"cd ..\.cargo\",
+//             r"cd ..\.cargo\",
-            commands![
+//             commands![
-                (cd bare(r"..\.cargo\"))
+//                 (cd bare(r"..\.cargo\"))
-            ],
+//             ],
-        );
+//         );
 
-        assert_parse(
+//         assert_parse(
-            "ls | where size < 1KB",
+//             "ls | where size < 1KB",
-            commands![
+//             commands![
-                (ls) | (where binary((&|b| b.bare("size"), &|b| b.op("<"), &|b| b.unit((1, "KB")))))
+//                 (ls) | (where binary((&|b| b.bare("size"), &|b| b.op("<"), &|b| b.unit((1, "KB")))))
-            ],
+//             ],
-        );
+//         );
 
-        assert_parse(
+//         assert_parse(
-            "ls | where { $it.size > 100 }",
+//             "ls | where { $it.size > 100 }",
-            commands![
+//             commands![
-                (ls) | (where block(&|b| b.binary((&|b| b.path((&|b| b.var("it"), vec!["size"])), &|b| b.op(">"), &|b| b.int(100)))))
+//                 (ls) | (where block(&|b| b.binary((&|b| b.path((&|b| b.var("it"), vec!["size"])), &|b| b.op(">"), &|b| b.int(100)))))
-            ],
+//             ],
-        )
+//         )
-    }
+//     }
 
-    use crate::parser::ast::{ModuleBuilder, RawItem};
+//     use crate::parser::ast::{ModuleBuilder, RawItem};
-    use crate::parser::lexer::Spanned;
+//     use crate::parser::lexer::Spanned;
 
-}
+// }

src/parser/hir.rs

@@ -0,0 +1,93 @@
+crate mod baseline_parse;
+crate mod baseline_parse_tokens;
+crate mod binary;
+crate mod named;
+crate mod path;
+
+use crate::parser::{Span, Spanned, Unit};
+use derive_new::new;
+use getset::Getters;
+
+crate use baseline_parse::baseline_parse_single_token;
+crate use baseline_parse_tokens::{
+    baseline_parse_next_expr, baseline_parse_tokens, ExpressionKindHint,
+};
+crate use binary::Binary;
+crate use named::NamedArguments;
+crate use path::Path;
+
+#[derive(Debug, Clone, Eq, PartialEq, Getters, new)]
+pub struct Call {
+    #[get = "crate"]
+    head: Box<Expression>,
+    #[get = "crate"]
+    positional: Option<Vec<Expression>>,
+    #[get = "crate"]
+    named: Option<NamedArguments>,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub enum RawExpression {
+    Literal(Literal),
+    Variable(Variable),
+    Binary(Box<Binary>),
+    Block(Box<Expression>),
+    Path(Box<Path>),
+
+    #[allow(unused)]
+    Boolean(bool),
+}
+
+pub type Expression = Spanned<RawExpression>;
+
+impl Expression {
+    fn int(i: impl Into<i64>, span: impl Into<Span>) -> Expression {
+        Spanned::from_item(RawExpression::Literal(Literal::Integer(i.into())), span)
+    }
+
+    fn size(i: impl Into<i64>, unit: impl Into<Unit>, span: impl Into<Span>) -> Expression {
+        Spanned::from_item(
+            RawExpression::Literal(Literal::Size(i.into(), unit.into())),
+            span,
+        )
+    }
+
+    fn string(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
+        Spanned::from_item(
+            RawExpression::Literal(Literal::String(inner.into())),
+            outer.into(),
+        )
+    }
+
+    fn bare(span: impl Into<Span>) -> Expression {
+        Spanned::from_item(RawExpression::Literal(Literal::Bare), span.into())
+    }
+
+    fn variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
+        Spanned::from_item(
+            RawExpression::Variable(Variable::Other(inner.into())),
+            outer.into(),
+        )
+    }
+
+    fn it_variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
+        Spanned::from_item(
+            RawExpression::Variable(Variable::It(inner.into())),
+            outer.into(),
+        )
+    }
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub enum Literal {
+    Integer(i64),
+    Size(i64, Unit),
+    String(Span),
+    Bare,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub enum Variable {
+    It(Span),
+    Other(Span),
+}

src/parser/hir/baseline_parse.rs

@@ -0,0 +1,30 @@
+use crate::errors::ShellError;
+use crate::parser::{hir, CommandRegistry, RawToken, Token, TokenNode};
+
+// pub fn baseline_parse_token(
+//     token_node: TokenNode,
+//     _registry: &dyn CommandRegistry,
+// ) -> Result<hir::Expression, ShellError> {
+//     match token_node {
+//         TokenNode::Token(token) => Ok(baseline_parse_single_token(token)),
+//         TokenNode::Call(_call) => Err(ShellError::unimplemented("baseline_parse Call")),
+//         TokenNode::Delimited(_delimited) => {
+//             Err(ShellError::unimplemented("baseline_parse Delimited"))
+//         }
+//         TokenNode::Pipeline(_pipeline) => Err(ShellError::unimplemented("baseline_parse Pipeline")),
+//         TokenNode::Path(_path) => Err(ShellError::unimplemented("baseline_parse Path")),
+//     }
+// }
+
+pub fn baseline_parse_single_token(token: &Token, source: &str) -> hir::Expression {
+    match *token.item() {
+        RawToken::Integer(int) => hir::Expression::int(int, token.span),
+        RawToken::Size(int, unit) => hir::Expression::size(int, unit, token.span),
+        RawToken::String(span) => hir::Expression::string(span, token.span),
+        RawToken::Variable(span) if span.slice(source) == "it" => {
+            hir::Expression::it_variable(span, token.span)
+        }
+        RawToken::Variable(span) => hir::Expression::variable(span, token.span),
+        RawToken::Bare => hir::Expression::bare(token.span),
+    }
+}

src/parser/hir/baseline_parse_tokens.rs

@@ -0,0 +1,177 @@
+use crate::errors::ShellError;
+use crate::parser::registry::CommandRegistry;
+use crate::parser::{hir, hir::baseline_parse_single_token, Span, Spanned, TokenNode};
+
+pub fn baseline_parse_tokens(
+    token_nodes: &[TokenNode],
+    registry: &dyn CommandRegistry,
+    source: &str,
+) -> Result<Vec<hir::Expression>, ShellError> {
+    let mut exprs: Vec<hir::Expression> = vec![];
+    let mut rest = token_nodes;
+
+    loop {
+        if rest.len() == 0 {
+            break;
+        }
+
+        let (expr, remainder) = baseline_parse_next_expr(rest, registry, source, None)?;
+        exprs.push(expr);
+        rest = remainder;
+    }
+
+    Ok(exprs)
+}
+
+#[derive(Debug)]
+pub enum ExpressionKindHint {
+    Literal,
+    Variable,
+    Binary,
+    Block,
+    Boolean,
+}
+
+pub fn baseline_parse_next_expr(
+    token_nodes: &'nodes [TokenNode],
+    _registry: &dyn CommandRegistry,
+    source: &str,
+    coerce_hint: Option<ExpressionKindHint>,
+) -> Result<(hir::Expression, &'nodes [TokenNode]), ShellError> {
+    println!(
+        "baseline_parse_next_expr {:?} - {:?}",
+        token_nodes, coerce_hint
+    );
+
+    let mut tokens = token_nodes.iter().peekable();
+
+    let first = next_token(&mut tokens);
+
+    let first = match first {
+        None => return Err(ShellError::unimplemented("Expected token, found none")),
+        Some(token) => baseline_parse_semantic_token(token, source)?,
+    };
+
+    let possible_op = tokens.peek();
+
+    let op = match possible_op {
+        Some(TokenNode::Operator(op)) => op,
+        _ => return Ok((first, &token_nodes[1..])),
+    };
+
+    tokens.next();
+
+    let second = match tokens.next() {
+        None => {
+            return Err(ShellError::unimplemented(
+                "Expected op followed by another expr, found nothing",
+            ))
+        }
+        Some(token) => baseline_parse_semantic_token(token, source)?,
+    };
+
+    // We definitely have a binary expression here -- let's see if we should coerce it into a block
+
+    match coerce_hint {
+        None => {
+            let span = (first.span.start, second.span.end);
+            let binary = hir::Binary::new(first, *op, second);
+            let binary = hir::RawExpression::Binary(Box::new(binary));
+            let binary = Spanned::from_item(binary, span);
+
+            Ok((binary, &token_nodes[3..]))
+        }
+
+        Some(hint) => match hint {
+            ExpressionKindHint::Block => {
+                let span = (first.span.start, second.span.end);
+
+                let string: Spanned<String> = match first {
+                    Spanned {
+                        item: hir::RawExpression::Literal(hir::Literal::Bare),
+                        span,
+                    } => Spanned::from_item(span.slice(source).to_string(), span),
+                    Spanned {
+                        item: hir::RawExpression::Literal(hir::Literal::String(inner)),
+                        span,
+                    } => Spanned::from_item(inner.slice(source).to_string(), span),
+                    _ => {
+                        return Err(ShellError::unimplemented(
+                            "The first part of a block must be a string",
+                        ))
+                    }
+                };
+
+                let path = hir::Path::new(
+                    Spanned::from_item(
+                        // TODO: Deal with synthetic nodes that have no representation at all in source
+                        hir::RawExpression::Variable(hir::Variable::It(Span::from((0, 0)))),
+                        (0, 0),
+                    ),
+                    vec![string],
+                );
+                let path = hir::RawExpression::Path(Box::new(path));
+                let path = Spanned::from_item(path, first.span);
+
+                let binary = hir::Binary::new(path, *op, second);
+                let binary = hir::RawExpression::Binary(Box::new(binary));
+                let binary = Spanned::from_item(binary, span);
+
+                let block = hir::RawExpression::Block(Box::new(binary));
+                let block = Spanned::from_item(block, span);
+
+                Ok((block, &token_nodes[3..]))
+            }
+
+            other => unimplemented!("coerce hint {:?}", other),
+        },
+    }
+}
+
+pub fn baseline_parse_semantic_token(
+    token: &TokenNode,
+    source: &str,
+) -> Result<hir::Expression, ShellError> {
+    match token {
+        TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)),
+        TokenNode::Call(call) => unimplemented!(),
+        TokenNode::Delimited(delimited) => unimplemented!(),
+        TokenNode::Pipeline(pipeline) => unimplemented!(),
+        TokenNode::Operator(_op) => unreachable!(),
+        TokenNode::Flag(flag) => unimplemented!(),
+        TokenNode::Identifier(_span) => unreachable!(),
+        TokenNode::Whitespace(_span) => unreachable!(),
+        TokenNode::Error(error) => Err(*error.item.clone()),
+        TokenNode::Path(path) => unimplemented!(),
+    }
+}
+
+fn next_token(nodes: &mut impl Iterator<Item = &'a TokenNode>) -> Option<&'a TokenNode> {
+    loop {
+        match nodes.next() {
+            Some(TokenNode::Whitespace(_)) => continue,
+            other => return other,
+        }
+    }
+}
+
+fn baseline_parse_token(
+    token_node: &TokenNode,
+    _registry: &dyn CommandRegistry,
+    source: &str,
+) -> Result<hir::Expression, ShellError> {
+    match token_node {
+        TokenNode::Token(token) => Ok(hir::baseline_parse_single_token(token, source)),
+        TokenNode::Call(_call) => Err(ShellError::unimplemented("baseline_parse Call")),
+        TokenNode::Delimited(_delimited) => {
+            Err(ShellError::unimplemented("baseline_parse Delimited"))
+        }
+        TokenNode::Pipeline(_pipeline) => Err(ShellError::unimplemented("baseline_parse Pipeline")),
+        TokenNode::Path(_path) => Err(ShellError::unimplemented("baseline_parse Path")),
+        TokenNode::Operator(_op) => Err(ShellError::unimplemented("baseline_parse Operator")),
+        TokenNode::Flag(_op) => Err(ShellError::unimplemented("baseline_parse Flag")),
+        TokenNode::Identifier(_op) => Err(ShellError::unimplemented("baseline_parse Identifier")),
+        TokenNode::Whitespace(_op) => Err(ShellError::unimplemented("baseline_parse Whitespace")),
+        TokenNode::Error(err) => Err(*err.item.clone()),
+    }
+}

src/parser/hir/binary.rs

@@ -0,0 +1,11 @@
+use crate::parser::{hir::Expression, Operator, Spanned};
+use derive_new::new;
+use getset::Getters;
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, new)]
+#[get = "crate"]
+pub struct Binary {
+    left: Expression,
+    op: Spanned<Operator>,
+    right: Expression,
+}

src/parser/hir/named.rs

@@ -0,0 +1,44 @@
+use crate::parser::hir::Expression;
+use crate::parser::{Flag, Span};
+use derive_new::new;
+use indexmap::IndexMap;
+use log::trace;
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum NamedValue {
+    AbsentSwitch,
+    PresentSwitch(Span),
+    AbsentValue,
+    Value(Expression),
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, new)]
+pub struct NamedArguments {
+    #[new(default)]
+    crate named: IndexMap<String, NamedValue>,
+}
+
+impl NamedArguments {
+    pub fn insert_switch(&mut self, name: impl Into<String>, switch: Option<Flag>) {
+        let name = name.into();
+        trace!("Inserting switch -- {} = {:?}", name, switch);
+
+        match switch {
+            None => self.named.insert(name.into(), NamedValue::AbsentSwitch),
+            Some(flag) => self
+                .named
+                .insert(name, NamedValue::PresentSwitch(*flag.name())),
+        };
+    }
+
+    pub fn insert_optional(&mut self, name: impl Into<String>, expr: Option<Expression>) {
+        match expr {
+            None => self.named.insert(name.into(), NamedValue::AbsentValue),
+            Some(expr) => self.named.insert(name.into(), NamedValue::Value(expr)),
+        };
+    }
+
+    pub fn insert_mandatory(&mut self, name: impl Into<String>, expr: Expression) {
+        self.named.insert(name.into(), NamedValue::Value(expr));
+    }
+}

src/parser/hir/path.rs

@@ -0,0 +1,10 @@
+use crate::parser::{hir::Expression, Operator, Spanned};
+use derive_new::new;
+use getset::Getters;
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, new)]
+#[get = "crate"]
+pub struct Path {
+    head: Expression,
+    tail: Vec<Spanned<String>>,
+}

src/parser/parse2.rs

@@ -1,9 +1,14 @@
+crate mod call_node;
+crate mod files;
 crate mod flag;
 crate mod operator;
 crate mod parser;
 crate mod span;
+crate mod text;
 crate mod token_tree;
 crate mod token_tree_builder;
 crate mod tokens;
 crate mod unit;
 crate mod util;
+
+crate use token_tree::{PipelineElement, TokenNode};

src/parser/parse2/call_node.rs

@@ -0,0 +1,26 @@
+use crate::parser::TokenNode;
+use getset::Getters;
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
+pub struct CallNode {
+    #[get = "crate"]
+    head: Box<TokenNode>,
+    #[get = "crate"]
+    children: Option<Vec<TokenNode>>,
+}
+
+impl CallNode {
+    pub fn new(head: Box<TokenNode>, children: Vec<TokenNode>) -> CallNode {
+        if children.len() == 0 {
+            CallNode {
+                head,
+                children: None,
+            }
+        } else {
+            CallNode {
+                head,
+                children: Some(children),
+            }
+        }
+    }
+}

src/parser/parse2/files.rs

@@ -0,0 +1,77 @@
+use crate::parser::parse2::span::Span;
+use derive_new::new;
+use language_reporting::{FileName, Location};
+
+#[derive(new, Debug, Clone)]
+pub struct Files {
+    snippet: String,
+}
+
+impl language_reporting::ReportingFiles for Files {
+    type Span = Span;
+    type FileId = usize;
+
+    fn byte_span(
+        &self,
+        _file: Self::FileId,
+        from_index: usize,
+        to_index: usize,
+    ) -> Option<Self::Span> {
+        Some(Span::from((from_index, to_index)))
+    }
+    fn file_id(&self, _span: Self::Span) -> Self::FileId {
+        0
+    }
+    fn file_name(&self, _file: Self::FileId) -> FileName {
+        FileName::Verbatim(format!("<eval>"))
+    }
+    fn byte_index(&self, _file: Self::FileId, _line: usize, _column: usize) -> Option<usize> {
+        unimplemented!("byte_index")
+    }
+    fn location(&self, _file: Self::FileId, byte_index: usize) -> Option<Location> {
+        let source = &self.snippet;
+        let mut seen_lines = 0;
+        let mut seen_bytes = 0;
+
+        for (pos, _) in source.match_indices('\n') {
+            if pos > byte_index {
+                return Some(language_reporting::Location::new(
+                    seen_lines,
+                    byte_index - seen_bytes,
+                ));
+            } else {
+                seen_lines += 1;
+                seen_bytes = pos;
+            }
+        }
+
+        if seen_lines == 0 {
+            Some(language_reporting::Location::new(0, byte_index))
+        } else {
+            None
+        }
+    }
+    fn line_span(&self, _file: Self::FileId, lineno: usize) -> Option<Self::Span> {
+        let source = &self.snippet;
+        let mut seen_lines = 0;
+        let mut seen_bytes = 0;
+
+        for (pos, _) in source.match_indices('\n') {
+            if seen_lines == lineno {
+                return Some(Span::from((seen_bytes, pos)));
+            } else {
+                seen_lines += 1;
+                seen_bytes = pos + 1;
+            }
+        }
+
+        if seen_lines == 0 {
+            Some(Span::from((0, self.snippet.len() - 1)))
+        } else {
+            None
+        }
+    }
+    fn source(&self, span: Self::Span) -> Option<String> {
+        Some(self.snippet[span.start..span.end].to_string())
+    }
+}

src/parser/parse2/flag.rs

@@ -1,7 +1,17 @@
+use crate::parser::Span;
+use derive_new::new;
+use getset::Getters;
 use serde_derive::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
-pub enum Flag {
+pub enum FlagKind {
     Shorthand,
     Longhand,
 }
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Getters, new)]
+#[get = "crate"]
+pub struct Flag {
+    kind: FlagKind,
+    name: Span,
+}

src/parser/parse2/operator.rs

@@ -12,6 +12,7 @@ pub enum Operator {
 }
 
 impl Operator {
+    #[allow(unused)]
     pub fn print(&self) -> String {
         self.as_str().to_string()
     }

src/parser/parse2/span.rs

@@ -1,7 +1,8 @@
 use derive_new::new;
-use std::ops::Range;
+use getset::Getters;
 
-#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Getters)]
+#[get = "crate"]
 pub struct Spanned<T> {
     crate span: Span,
     crate item: T,
@@ -16,20 +17,6 @@ impl<T> std::ops::Deref for Spanned<T> {
 }
 
 impl<T> Spanned<T> {
-    crate fn from_nom<U>(
-        item: T,
-        start: nom_locate::LocatedSpan<U>,
-        end: nom_locate::LocatedSpan<U>,
-    ) -> Spanned<T> {
-        let start = start.offset;
-        let end = end.offset;
-
-        Spanned {
-            span: Span::from((start, end)),
-            item,
-        }
-    }
-
     crate fn from_item(item: T, span: impl Into<Span>) -> Spanned<T> {
         Spanned {
             span: span.into(),
@@ -43,6 +30,19 @@ impl<T> Spanned<T> {
         let mapped = input(item);
         Spanned { span, item: mapped }
     }
+
+    crate fn copy_span<U>(&self, output: U) -> Spanned<U> {
+        let Spanned { span, item } = self;
+
+        Spanned {
+            span: *span,
+            item: output,
+        }
+    }
+
+    pub fn source(&self, source: &'source str) -> &'source str {
+        self.span().slice(source)
+    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)]
@@ -52,6 +52,21 @@ pub struct Span {
     // source: &'source str,
 }
 
+impl From<&Span> for Span {
+    fn from(input: &Span) -> Span {
+        *input
+    }
+}
+
+impl From<nom_locate::LocatedSpan<&str>> for Span {
+    fn from(input: nom_locate::LocatedSpan<&str>) -> Span {
+        Span {
+            start: input.offset,
+            end: input.offset + input.fragment.len(),
+        }
+    }
+}
+
 impl<T> From<(nom_locate::LocatedSpan<T>, nom_locate::LocatedSpan<T>)> for Span {
     fn from(input: (nom_locate::LocatedSpan<T>, nom_locate::LocatedSpan<T>)) -> Span {
         Span {
@@ -80,12 +95,8 @@ impl From<&std::ops::Range<usize>> for Span {
 }
 
 impl Span {
-    fn new(range: &Range<usize>) -> Span {
+    pub fn slice(&self, source: &'source str) -> &'source str {
-        Span {
+        &source[self.start..self.end]
-            start: range.start,
-            end: range.end,
-            // source,
-        }
     }
 }
 

src/parser/parse2/text.rs

@@ -0,0 +1,204 @@
+use std::cmp::Ordering;
+use std::hash::Hash;
+use std::hash::Hasher;
+use std::ops::Range;
+use std::sync::Arc;
+
+/// A "Text" is like a string except that it can be cheaply cloned.
+/// You can also "extract" subtexts quite cheaply. You can also deref
+/// an `&Text` into a `&str` for interoperability.
+///
+/// Used to represent the value of an input file.
+#[derive(Clone)]
+pub struct Text {
+    text: Arc<String>,
+    start: usize,
+    end: usize,
+}
+
+impl Text {
+    /// Modifies this restrict to a subset of its current range.
+    pub fn select(&mut self, range: Range<usize>) {
+        let len = range.end - range.start;
+        let new_start = self.start + range.start;
+        let new_end = new_start + len;
+        assert!(new_end <= self.end);
+
+        self.start = new_start;
+        self.end = new_end;
+    }
+
+    /// Extract a new `Text` that is a subset of an old `Text`
+    /// -- `text.extract(1..3)` is similar to `&foo[1..3]` except that
+    /// it gives back an owned value instead of a borrowed value.
+    pub fn extract(&self, range: Range<usize>) -> Self {
+        let mut result = self.clone();
+        result.select(range);
+        result
+    }
+}
+
+impl From<Arc<String>> for Text {
+    fn from(text: Arc<String>) -> Self {
+        let end = text.len();
+        Self {
+            text,
+            start: 0,
+            end,
+        }
+    }
+}
+
+impl AsRef<str> for Text {
+    fn as_ref(&self) -> &str {
+        &*self
+    }
+}
+
+impl From<String> for Text {
+    fn from(text: String) -> Self {
+        Text::from(Arc::new(text))
+    }
+}
+
+impl From<&String> for Text {
+    fn from(text: &String) -> Self {
+        Text::from(text.to_string())
+    }
+}
+
+impl From<&str> for Text {
+    fn from(text: &str) -> Self {
+        Text::from(text.to_string())
+    }
+}
+
+impl std::borrow::Borrow<str> for Text {
+    fn borrow(&self) -> &str {
+        &*self
+    }
+}
+
+impl std::ops::Deref for Text {
+    type Target = str;
+
+    fn deref(&self) -> &str {
+        &self.text[self.start..self.end]
+    }
+}
+
+impl std::fmt::Display for Text {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        <str as std::fmt::Display>::fmt(self, fmt)
+    }
+}
+
+impl std::fmt::Debug for Text {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        <str as std::fmt::Debug>::fmt(self, fmt)
+    }
+}
+
+impl PartialEq<Text> for Text {
+    fn eq(&self, other: &Text) -> bool {
+        let this: &str = self;
+        let other: &str = other;
+        this == other
+    }
+}
+
+impl Eq for Text {}
+
+impl PartialEq<str> for Text {
+    fn eq(&self, other: &str) -> bool {
+        let this: &str = self;
+        this == other
+    }
+}
+
+impl PartialEq<String> for Text {
+    fn eq(&self, other: &String) -> bool {
+        let this: &str = self;
+        let other: &str = other;
+        this == other
+    }
+}
+
+impl PartialEq<Text> for str {
+    fn eq(&self, other: &Text) -> bool {
+        other == self
+    }
+}
+
+impl PartialEq<Text> for String {
+    fn eq(&self, other: &Text) -> bool {
+        other == self
+    }
+}
+
+impl<T: ?Sized> PartialEq<&T> for Text
+where
+    Text: PartialEq<T>,
+{
+    fn eq(&self, other: &&T) -> bool {
+        self == *other
+    }
+}
+
+impl Hash for Text {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        <str as Hash>::hash(self, state)
+    }
+}
+
+impl PartialOrd<Text> for Text {
+    fn partial_cmp(&self, other: &Text) -> Option<Ordering> {
+        let this: &str = self;
+        let other: &str = other;
+        this.partial_cmp(other)
+    }
+}
+
+impl Ord for Text {
+    fn cmp(&self, other: &Text) -> Ordering {
+        let this: &str = self;
+        let other: &str = other;
+        this.cmp(other)
+    }
+}
+
+impl PartialOrd<str> for Text {
+    fn partial_cmp(&self, other: &str) -> Option<Ordering> {
+        let this: &str = self;
+        this.partial_cmp(other)
+    }
+}
+
+impl PartialOrd<String> for Text {
+    fn partial_cmp(&self, other: &String) -> Option<Ordering> {
+        let this: &str = self;
+        let other: &str = other;
+        this.partial_cmp(other)
+    }
+}
+
+impl PartialOrd<Text> for str {
+    fn partial_cmp(&self, other: &Text) -> Option<Ordering> {
+        other.partial_cmp(self)
+    }
+}
+
+impl PartialOrd<Text> for String {
+    fn partial_cmp(&self, other: &Text) -> Option<Ordering> {
+        other.partial_cmp(self)
+    }
+}
+
+impl<T: ?Sized> PartialOrd<&T> for Text
+where
+    Text: PartialOrd<T>,
+{
+    fn partial_cmp(&self, other: &&T) -> Option<Ordering> {
+        self.partial_cmp(*other)
+    }
+}

src/parser/parse2/token_tree.rs

@@ -1,14 +1,22 @@
-use crate::parser::parse2::{operator::*, span::*, tokens::*};
+use crate::errors::ShellError;
+use crate::parser::parse2::{call_node::*, flag::*, operator::*, span::*, tokens::*};
 use derive_new::new;
 use enum_utils::FromStr;
+use getset::Getters;
 
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
 pub enum TokenNode {
     Token(Token),
+    #[allow(unused)]
     Call(Spanned<CallNode>),
     Delimited(Spanned<DelimitedNode>),
-    Pipeline(Spanned<Vec<TokenNode>>),
+    Pipeline(Spanned<Vec<PipelineElement>>),
-    Binary(Spanned<BinaryNode>),
+    Operator(Spanned<Operator>),
+    Flag(Spanned<Flag>),
+    Identifier(Span),
+    Whitespace(Span),
+    #[allow(unused)]
+    Error(Spanned<Box<ShellError>>),
     Path(Spanned<PathNode>),
 }
 
@@ -19,10 +27,71 @@ impl TokenNode {
             TokenNode::Call(s) => s.span,
             TokenNode::Delimited(s) => s.span,
             TokenNode::Pipeline(s) => s.span,
-            TokenNode::Binary(s) => s.span,
+            TokenNode::Operator(s) => s.span,
+            TokenNode::Flag(s) => s.span,
+            TokenNode::Identifier(s) => *s,
+            TokenNode::Whitespace(s) => *s,
+            TokenNode::Error(s) => s.span,
             TokenNode::Path(s) => s.span,
         }
     }
+
+    pub fn as_external_arg(&self, source: &str) -> String {
+        self.span().slice(source).to_string()
+    }
+
+    pub fn source(&self, source: &'source str) -> &'source str {
+        self.span().slice(source)
+    }
+
+    pub fn is_ws(&self) -> bool {
+        match self {
+            TokenNode::Whitespace(_) => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_bare(&self) -> bool {
+        match self {
+            TokenNode::Token(Spanned {
+                item: RawToken::Bare,
+                ..
+            }) => true,
+            _ => false,
+        }
+    }
+
+    crate fn as_string(&self, source: &str) -> Option<Spanned<String>> {
+        match self {
+            TokenNode::Token(Spanned {
+                item: RawToken::Bare,
+                span,
+            }) => Some(Spanned::from_item(span.slice(source).to_string(), span)),
+            TokenNode::Token(Spanned {
+                item: RawToken::String(inner),
+                span,
+            }) => Some(Spanned::from_item(inner.slice(source).to_string(), span)),
+            _ => None,
+        }
+    }
+
+    crate fn as_flag(&self, value: &str, source: &str) -> Option<Spanned<Flag>> {
+        match self {
+            TokenNode::Flag(
+                flag @ Spanned {
+                    item: Flag { .. }, ..
+                },
+            ) if value == flag.name().slice(source) => Some(*flag),
+            _ => None,
+        }
+    }
+
+    pub fn as_pipeline(&self) -> Result<Vec<PipelineElement>, ShellError> {
+        match self {
+            TokenNode::Pipeline(Spanned { item, .. }) => Ok(item.clone()),
+            _ => Err(ShellError::string("unimplemented")),
+        }
+    }
 }
 
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)]
@@ -31,12 +100,6 @@ pub struct DelimitedNode {
     children: Vec<TokenNode>,
 }
 
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)]
-pub struct CallNode {
-    head: Box<TokenNode>,
-    children: Vec<TokenNode>,
-}
-
 #[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, FromStr)]
 pub enum Delimiter {
     Paren,
@@ -50,9 +113,26 @@ pub struct PathNode {
     tail: Vec<TokenNode>,
 }
 
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)]
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
-pub struct BinaryNode {
+pub struct PipelineElement {
-    left: Box<TokenNode>,
+    pre_ws: Option<Span>,
-    op: Operator,
+    #[get = "crate"]
-    right: Box<TokenNode>,
+    call: Spanned<CallNode>,
+    post_ws: Option<Span>,
+}
+
+impl PipelineElement {
+    crate fn span(&self) -> Span {
+        let start = match self.pre_ws {
+            None => self.call.span.start,
+            Some(span) => span.start,
+        };
+
+        let end = match self.post_ws {
+            None => self.call.span.end,
+            Some(span) => span.end,
+        };
+
+        Span::from((start, end))
+    }
 }

src/parser/parse2/token_tree_builder.rs

@@ -1,11 +1,15 @@
-use crate::parser::parse2::flag::Flag;
+#[allow(unused)]
+use crate::prelude::*;
+
+use crate::parser::parse2::flag::{Flag, FlagKind};
 use crate::parser::parse2::operator::Operator;
 use crate::parser::parse2::span::{Span, Spanned};
 use crate::parser::parse2::token_tree::{
-    BinaryNode, CallNode, DelimitedNode, Delimiter, PathNode, TokenNode,
+    DelimitedNode, Delimiter, PathNode, PipelineElement, TokenNode,
 };
 use crate::parser::parse2::tokens::{RawToken, Token};
 use crate::parser::parse2::unit::Unit;
+use crate::parser::CallNode;
 use derive_new::new;
 
 #[derive(new)]
@@ -14,7 +18,10 @@ pub struct TokenTreeBuilder {
     pos: usize,
 }
 
-pub type CurriedToken = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Option<TokenNode> + 'static>;
+#[allow(unused)]
+pub type CurriedNode<T> = Box<dyn FnOnce(&mut TokenTreeBuilder) -> T + 'static>;
+pub type CurriedToken = Box<dyn FnOnce(&mut TokenTreeBuilder) -> TokenNode + 'static>;
+pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Spanned<CallNode> + 'static>;
 
 #[allow(unused)]
 impl TokenTreeBuilder {
@@ -23,35 +30,57 @@ impl TokenTreeBuilder {
         block(&mut builder)
     }
 
-    pub fn pipeline(input: Vec<CurriedToken>) -> CurriedToken {
+    pub fn pipeline(input: Vec<(Option<&str>, CurriedCall, Option<&str>)>) -> CurriedToken {
+        let input: Vec<(Option<String>, CurriedCall, Option<String>)> = input
+            .into_iter()
+            .map(|(pre, call, post)| {
+                (
+                    pre.map(|s| s.to_string()),
+                    call,
+                    post.map(|s| s.to_string()),
+                )
+            })
+            .collect();
+
         Box::new(move |b| {
             let start = b.pos;
 
-            let mut out = vec![];
+            let mut out: Vec<PipelineElement> = vec![];
 
             let mut input = input.into_iter();
-            let first = input
+            let (pre, call, post) = input
                 .next()
                 .expect("A pipeline must contain at least one element");
 
-            out.push(first(b).expect("The first element of a pipeline must not be whitespace"));
+            let pre_span = pre.map(|pre| b.consume(&pre));
+            let call = call(b);
+            let post_span = post.map(|post| b.consume(&post));
+            out.push(PipelineElement::new(
+                pre_span.map(Span::from),
+                call,
+                post_span.map(Span::from),
+            ));
 
-            for item in input {
+            for (pre, call, post) in input {
-                b.consume(" | ");
+                b.consume("|");
+                let pre_span = pre.map(|pre| b.consume(&pre));
+                let call = call(b);
+                let post_span = post.map(|post| b.consume(&post));
 
-                match item(b) {
+                out.push(PipelineElement::new(
-                    None => {}
+                    pre_span.map(Span::from),
-                    Some(v) => out.push(v),
+                    call,
-                }
+                    post_span.map(Span::from),
+                ));
             }
 
             let end = b.pos;
 
-            Some(TokenTreeBuilder::spanned_pipeline(out, (start, end)))
+            TokenTreeBuilder::spanned_pipeline(out, (start, end))
         })
     }
 
-    pub fn spanned_pipeline(input: Vec<TokenNode>, span: impl Into<Span>) -> TokenNode {
+    pub fn spanned_pipeline(input: Vec<PipelineElement>, span: impl Into<Span>) -> TokenNode {
         TokenNode::Pipeline(Spanned::from_item(input, span))
     }
 
@@ -63,15 +92,12 @@ impl TokenTreeBuilder {
 
             b.pos = end;
 
-            Some(TokenTreeBuilder::spanned_op(input, (start, end)))
+            TokenTreeBuilder::spanned_op(input, (start, end))
         })
     }
 
     pub fn spanned_op(input: impl Into<Operator>, span: impl Into<Span>) -> TokenNode {
-        TokenNode::Token(Spanned::from_item(
+        TokenNode::Operator(Spanned::from_item(input.into(), span.into()))
-            RawToken::Operator(input.into()),
-            span.into(),
-        ))
     }
 
     pub fn string(input: impl Into<String>) -> CurriedToken {
@@ -83,10 +109,7 @@ impl TokenTreeBuilder {
             let (_, end) = b.consume("\"");
             b.pos = end;
 
-            Some(TokenTreeBuilder::spanned_string(
+            TokenTreeBuilder::spanned_string((inner_start, inner_end), (start, end))
-                (inner_start, inner_end),
-                (start, end),
-            ))
         })
     }
 
@@ -104,7 +127,7 @@ impl TokenTreeBuilder {
             let (start, end) = b.consume(&input);
             b.pos = end;
 
-            Some(TokenTreeBuilder::spanned_bare((start, end)))
+            TokenTreeBuilder::spanned_bare((start, end))
         })
     }
 
@@ -119,7 +142,7 @@ impl TokenTreeBuilder {
             let (start, end) = b.consume(&int.to_string());
             b.pos = end;
 
-            Some(TokenTreeBuilder::spanned_int(int, (start, end)))
+            TokenTreeBuilder::spanned_int(int, (start, end))
         })
     }
 
@@ -136,7 +159,7 @@ impl TokenTreeBuilder {
             let (_, end) = b.consume(unit.as_str());
             b.pos = end;
 
-            Some(TokenTreeBuilder::spanned_size((int, unit), (start, end)))
+            TokenTreeBuilder::spanned_size((int, unit), (start, end))
         })
     }
 
@@ -152,22 +175,19 @@ impl TokenTreeBuilder {
     pub fn path(head: CurriedToken, tail: Vec<CurriedToken>) -> CurriedToken {
         Box::new(move |b| {
             let start = b.pos;
-            let head = head(b).expect("The head of a path must not be whitespace");
+            let head = head(b);
 
             let mut output = vec![];
 
             for item in tail {
                 b.consume(".");
 
-                match item(b) {
+                output.push(item(b));
-                    None => {}
-                    Some(v) => output.push(v),
-                };
             }
 
             let end = b.pos;
 
-            Some(TokenTreeBuilder::spanned_path((head, output), (start, end)))
+            TokenTreeBuilder::spanned_path((head, output), (start, end))
         })
     }
 
@@ -185,10 +205,7 @@ impl TokenTreeBuilder {
             let (start, _) = b.consume("$");
             let (inner_start, end) = b.consume(&input);
 
-            Some(TokenTreeBuilder::spanned_var(
+            TokenTreeBuilder::spanned_var((inner_start, end), (start, end))
-                (inner_start, end),
-                (start, end),
-            ))
         })
     }
 
@@ -206,16 +223,13 @@ impl TokenTreeBuilder {
             let (start, _) = b.consume("--");
             let (inner_start, end) = b.consume(&input);
 
-            Some(TokenTreeBuilder::spanned_flag(
+            TokenTreeBuilder::spanned_flag((inner_start, end), (start, end))
-                (inner_start, end),
-                (start, end),
-            ))
         })
     }
 
     pub fn spanned_flag(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
-        TokenNode::Token(Spanned::from_item(
+        TokenNode::Flag(Spanned::from_item(
-            RawToken::Flag(Flag::Longhand, input.into()),
+            Flag::new(FlagKind::Longhand, input.into()),
             span.into(),
         ))
     }
@@ -227,16 +241,13 @@ impl TokenTreeBuilder {
             let (start, _) = b.consume("-");
             let (inner_start, end) = b.consume(&input);
 
-            Some(TokenTreeBuilder::spanned_shorthand(
+            TokenTreeBuilder::spanned_shorthand((inner_start, end), (start, end))
-                (inner_start, end),
-                (start, end),
-            ))
         })
     }
 
     pub fn spanned_shorthand(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
-        TokenNode::Token(Spanned::from_item(
+        TokenNode::Flag(Spanned::from_item(
-            RawToken::Flag(Flag::Shorthand, input.into()),
+            Flag::new(FlagKind::Shorthand, input.into()),
             span.into(),
         ))
     }
@@ -246,45 +257,42 @@ impl TokenTreeBuilder {
 
         Box::new(move |b| {
             let (start, end) = b.consume(&input);
-            Some(TokenTreeBuilder::spanned_ident((start, end)))
+            TokenTreeBuilder::spanned_ident((start, end))
         })
     }
 
     pub fn spanned_ident(span: impl Into<Span>) -> TokenNode {
-        TokenNode::Token(Spanned::from_item(RawToken::Identifier, span.into()))
+        TokenNode::Identifier(span.into())
     }
 
-    pub fn call(head: CurriedToken, input: Vec<CurriedToken>) -> CurriedToken {
+    pub fn call(head: CurriedToken, input: Vec<CurriedToken>) -> CurriedCall {
         Box::new(move |b| {
             let start = b.pos;
 
-            let head_node = head(b).expect("The head of a command must not be whitespace");
+            let head_node = head(b);
 
-            let mut tail_nodes = vec![];
+            let mut nodes = vec![head_node];
             for item in input {
-                match item(b) {
+                nodes.push(item(b));
-                    None => {}
-                    Some(v) => tail_nodes.push(v),
-                };
             }
 
             let end = b.pos;
 
-            Some(TokenTreeBuilder::spanned_call(
+            TokenTreeBuilder::spanned_call(nodes, (start, end))
-                (head_node, Some(tail_nodes)),
-                (start, end),
-            ))
         })
     }
 
-    pub fn spanned_call(
+    pub fn spanned_call(input: Vec<TokenNode>, span: impl Into<Span>) -> Spanned<CallNode> {
-        input: (TokenNode, Option<Vec<TokenNode>>),
+        if input.len() == 0 {
-        span: impl Into<Span>,
+            panic!("BUG: spanned call (TODO)")
-    ) -> TokenNode {
+        }
-        TokenNode::Call(Spanned::from_item(
+
-            CallNode::new(Box::new(input.0), input.1.unwrap_or_else(|| vec![])),
+        let mut input = input.into_iter();
-            span,
+
-        ))
+        let head = input.next().unwrap();
+        let tail = input.collect();
+
+        Spanned::from_item(CallNode::new(Box::new(head), tail), span)
     }
 
     pub fn parens(input: Vec<CurriedToken>) -> CurriedToken {
@@ -292,15 +300,12 @@ impl TokenTreeBuilder {
             let (start, _) = b.consume("(");
             let mut output = vec![];
             for item in input {
-                match item(b) {
+                output.push(item(b));
-                    None => {}
-                    Some(v) => output.push(v),
-                };
             }
 
             let (_, end) = b.consume(")");
 
-            Some(TokenTreeBuilder::spanned_parens(output, (start, end)))
+            TokenTreeBuilder::spanned_parens(output, (start, end))
         })
     }
 
@@ -311,20 +316,38 @@ impl TokenTreeBuilder {
         ))
     }
 
+    pub fn square(input: Vec<CurriedToken>) -> CurriedToken {
+        Box::new(move |b| {
+            let (start, _) = b.consume("[");
+            let mut output = vec![];
+            for item in input {
+                output.push(item(b));
+            }
+
+            let (_, end) = b.consume("]");
+
+            TokenTreeBuilder::spanned_square(output, (start, end))
+        })
+    }
+
+    pub fn spanned_square(input: impl Into<Vec<TokenNode>>, span: impl Into<Span>) -> TokenNode {
+        TokenNode::Delimited(Spanned::from_item(
+            DelimitedNode::new(Delimiter::Square, input.into()),
+            span,
+        ))
+    }
+
     pub fn braced(input: Vec<CurriedToken>) -> CurriedToken {
         Box::new(move |b| {
             let (start, _) = b.consume("{ ");
             let mut output = vec![];
             for item in input {
-                match item(b) {
+                output.push(item(b));
-                    None => {}
-                    Some(v) => output.push(v),
-                };
             }
 
             let (_, end) = b.consume(" }");
 
-            Some(TokenTreeBuilder::spanned_brace(output, (start, end)))
+            TokenTreeBuilder::spanned_brace(output, (start, end))
         })
     }
 
@@ -335,50 +358,10 @@ impl TokenTreeBuilder {
         ))
     }
 
-    pub fn binary(
-        left: CurriedToken,
-        op: impl Into<Operator>,
-        right: CurriedToken,
-    ) -> CurriedToken {
-        let op = op.into();
-
-        Box::new(move |b| {
-            let start = b.pos;
-
-            let left = left(b).expect("The left side of a binary operation must not be whitespace");
-
-            b.consume(" ");
-
-            b.consume(op.as_str());
-
-            b.consume(" ");
-
-            let right =
-                right(b).expect("The right side of a binary operation must not be whitespace");
-
-            let end = b.pos;
-
-            Some(TokenTreeBuilder::spanned_binary(
-                (left, op, right),
-                (start, end),
-            ))
-        })
-    }
-
-    pub fn spanned_binary(
-        input: (impl Into<TokenNode>, Operator, impl Into<TokenNode>),
-        span: impl Into<Span>,
-    ) -> TokenNode {
-        TokenNode::Binary(Spanned::from_item(
-            BinaryNode::new(Box::new(input.0.into()), input.1, Box::new(input.2.into())),
-            span,
-        ))
-    }
-
     pub fn sp() -> CurriedToken {
         Box::new(|b| {
-            b.consume(" ");
+            let (start, end) = b.consume(" ");
-            None
+            TokenNode::Whitespace(Span::from((start, end)))
         })
     }
 
@@ -386,11 +369,17 @@ impl TokenTreeBuilder {
         let input = input.into();
 
         Box::new(move |b| {
-            b.consume(&input);
+            let (start, end) = b.consume(&input);
-            None
+            TokenTreeBuilder::spanned_ws((start, end))
         })
     }
 
+    pub fn spanned_ws(span: impl Into<Span>) -> TokenNode {
+        let span = span.into();
+
+        TokenNode::Whitespace(span.into())
+    }
+
     fn consume(&mut self, input: &str) -> (usize, usize) {
         let start = self.pos;
         self.pos += input.len();

src/parser/parse2/tokens.rs

@@ -7,12 +7,34 @@ use crate::parser::parse2::unit::*;
 pub enum RawToken {
     Integer(i64),
     Size(i64, Unit),
-    Operator(Operator),
     String(Span),
     Variable(Span),
-    Identifier,
     Bare,
-    Flag(Flag, Span),
 }
 
 pub type Token = Spanned<RawToken>;
+
+impl Token {
+    pub fn to_semantic_token(&self) -> Option<SemanticToken> {
+        let semantic_token = match self.item {
+            RawToken::Integer(int) => RawSemanticToken::Integer(int),
+            RawToken::Size(int, unit) => RawSemanticToken::Size(int, unit),
+            RawToken::String(span) => RawSemanticToken::String(span),
+            RawToken::Variable(span) => RawSemanticToken::Variable(span),
+            RawToken::Bare => RawSemanticToken::Bare,
+        };
+
+        Some(Spanned::from_item(semantic_token, self.span))
+    }
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub enum RawSemanticToken {
+    Integer(i64),
+    Size(i64, Unit),
+    String(Span),
+    Variable(Span),
+    Bare,
+}
+
+pub type SemanticToken = Spanned<RawSemanticToken>;

src/parser/parse2/unit.rs

@@ -12,10 +12,6 @@ pub enum Unit {
 }
 
 impl Unit {
-    pub fn print(&self) -> String {
-        self.as_str().to_string()
-    }
-
     pub fn as_str(&self) -> &str {
         match *self {
             Unit::B => "B",

src/parser/parse_command.rs

@@ -0,0 +1,250 @@
+use crate::errors::ShellError;
+use crate::parser::registry::{CommandConfig, CommandRegistry, NamedType};
+use crate::parser::{baseline_parse_tokens, CallNode, Spanned};
+use crate::parser::{
+    hir::{self, NamedArguments},
+    Flag, RawToken, TokenNode,
+};
+use log::trace;
+
+pub fn parse_command(
+    config: &CommandConfig,
+    registry: &dyn CommandRegistry,
+    call: &Spanned<CallNode>,
+    source: &str,
+) -> Result<hir::Call, ShellError> {
+    let Spanned { item: call, .. } = call;
+
+    trace!("Processing {:?}", config);
+
+    let head = parse_command_head(call.head())?;
+
+    let children: Option<Vec<TokenNode>> = call.children().as_ref().map(|nodes| {
+        nodes
+            .iter()
+            .cloned()
+            .filter(|node| match node {
+                TokenNode::Whitespace(_) => false,
+                _ => true,
+            })
+            .collect()
+    });
+
+    match parse_command_tail(&config, registry, children, source)? {
+        None => Ok(hir::Call::new(Box::new(head), None, None)),
+        Some((positional, named)) => Ok(hir::Call::new(Box::new(head), positional, named)),
+    }
+}
+
+fn parse_command_head(head: &TokenNode) -> Result<hir::Expression, ShellError> {
+    match head {
+        TokenNode::Token(
+            spanned @ Spanned {
+                item: RawToken::Bare,
+                ..
+            },
+        ) => Ok(spanned.map(|_| hir::RawExpression::Literal(hir::Literal::Bare))),
+
+        TokenNode::Token(Spanned {
+            item: RawToken::String(inner_span),
+            span,
+        }) => Ok(Spanned::from_item(
+            hir::RawExpression::Literal(hir::Literal::String(*inner_span)),
+            *span,
+        )),
+
+        other => Err(ShellError::unexpected(&format!(
+            "command head -> {:?}",
+            other
+        ))),
+    }
+}
+
+fn parse_command_tail(
+    config: &CommandConfig,
+    registry: &dyn CommandRegistry,
+    tail: Option<Vec<TokenNode>>,
+    source: &str,
+) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ShellError> {
+    let mut tail = match tail {
+        None => return Ok(None),
+        Some(tail) => tail,
+    };
+
+    let mut named = NamedArguments::new();
+
+    for (name, kind) in config.named() {
+        trace!("looking for {} : {:?}", name, kind);
+
+        match kind {
+            NamedType::Switch => {
+                let (rest, flag) = extract_switch(name, tail, source);
+
+                tail = rest;
+
+                named.insert_switch(name, flag);
+            }
+            NamedType::Mandatory(kind) => match extract_mandatory(name, tail, source) {
+                Err(err) => return Err(err), // produce a correct diagnostic
+                Ok((rest, pos, _flag)) => {
+                    let (expr, rest) = hir::baseline_parse_next_expr(
+                        &rest[pos..],
+                        registry,
+                        source,
+                        kind.to_coerce_hint(),
+                    )?;
+                    tail = rest.to_vec();
+
+                    named.insert_mandatory(name, expr);
+                }
+            },
+            NamedType::Optional(kind) => match extract_optional(name, tail, source) {
+                Err(err) => return Err(err), // produce a correct diagnostic
+                Ok((rest, Some((pos, _flag)))) => {
+                    let (expr, rest) = hir::baseline_parse_next_expr(
+                        &rest[pos..],
+                        registry,
+                        source,
+                        kind.to_coerce_hint(),
+                    )?;
+                    tail = rest.to_vec();
+
+                    named.insert_optional(name, Some(expr));
+                }
+
+                Ok((rest, None)) => {
+                    tail = rest;
+
+                    named.insert_optional(name, None);
+                }
+            },
+        };
+    }
+
+    let mut positional = vec![];
+    let mandatory = config.mandatory_positional();
+
+    for arg in mandatory {
+        if tail.len() == 0 {
+            return Err(ShellError::unimplemented("Missing mandatory argument"));
+        }
+
+        let (result, rest) =
+            hir::baseline_parse_next_expr(&tail, registry, source, arg.to_coerce_hint())?;
+
+        positional.push(result);
+
+        tail = rest.to_vec();
+    }
+
+    let optional = config.optional_positional();
+
+    for arg in optional {
+        if tail.len() == 0 {
+            break;
+        }
+
+        let (result, rest) =
+            hir::baseline_parse_next_expr(&tail, registry, source, arg.to_coerce_hint())?;
+
+        positional.push(result);
+
+        tail = rest.to_vec();
+    }
+
+    // TODO: Only do this if rest params are specified
+    let remainder = baseline_parse_tokens(&tail, registry, source)?;
+    positional.extend(remainder);
+
+    trace!("Constructed positional={:?} named={:?}", positional, named);
+
+    let positional = match positional {
+        positional if positional.len() == 0 => None,
+        positional => Some(positional),
+    };
+
+    let named = match named {
+        named if named.named.is_empty() => None,
+        named => Some(named),
+    };
+
+    trace!("Normalized positional={:?} named={:?}", positional, named);
+
+    Ok(Some((positional, named)))
+}
+
+fn extract_switch(
+    name: &str,
+    mut tokens: Vec<TokenNode>,
+    source: &str,
+) -> (Vec<TokenNode>, Option<Flag>) {
+    let pos = tokens
+        .iter()
+        .enumerate()
+        .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f)))
+        .nth(0);
+
+    match pos {
+        None => (tokens, None),
+        Some((pos, flag)) => {
+            tokens.remove(pos);
+            (tokens, Some(*flag))
+        }
+    }
+}
+
+fn extract_mandatory(
+    name: &str,
+    mut tokens: Vec<TokenNode>,
+    source: &str,
+) -> Result<(Vec<TokenNode>, usize, Flag), ShellError> {
+    let pos = tokens
+        .iter()
+        .enumerate()
+        .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f)))
+        .nth(0);
+
+    match pos {
+        None => Err(ShellError::unimplemented(
+            "Better error: mandatory flags must be present",
+        )),
+        Some((pos, flag)) => {
+            if tokens.len() <= pos {
+                return Err(ShellError::unimplemented(
+                    "Better errors: mandatory flags must be followed by values",
+                ));
+            }
+
+            tokens.remove(pos);
+
+            Ok((tokens, pos, *flag))
+        }
+    }
+}
+
+fn extract_optional(
+    name: &str,
+    mut tokens: Vec<TokenNode>,
+    source: &str,
+) -> Result<(Vec<TokenNode>, Option<(usize, Flag)>), ShellError> {
+    let pos = tokens
+        .iter()
+        .enumerate()
+        .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f)))
+        .nth(0);
+
+    match pos {
+        None => Ok((tokens, None)),
+        Some((pos, flag)) => {
+            if tokens.len() <= pos {
+                return Err(ShellError::unimplemented(
+                    "Better errors: optional flags must be followed by values",
+                ));
+            }
+
+            tokens.remove(pos);
+
+            Ok((tokens, Some((pos, *flag))))
+        }
+    }
+}

src/parser/registry.rs

@@ -1,7 +1,11 @@
-use crate::evaluate::{evaluate_expr, Scope};
+use crate::evaluate::{evaluate_baseline_expr, Scope};
-use crate::parser::lexer::Spanned;
+use crate::parser::{hir, hir::ExpressionKindHint, parse_command, CallNode, Spanned};
 use crate::prelude::*;
+use derive_new::new;
+use getset::Getters;
 use indexmap::IndexMap;
+use log::trace;
+use std::fmt;
 
 #[allow(unused)]
 #[derive(Debug)]
@@ -14,13 +18,18 @@ pub enum NamedType {
 #[derive(Debug)]
 pub enum NamedValue {
     Single,
-    Tuple,
 
     #[allow(unused)]
     Block,
+}
 
-    #[allow(unused)]
+impl NamedValue {
-    Array,
+    crate fn to_coerce_hint(&self) -> Option<ExpressionKindHint> {
+        match self {
+            NamedValue::Single => None,
+            NamedValue::Block => Some(ExpressionKindHint::Block),
+        }
+    }
 }
 
 #[allow(unused)]
@@ -31,61 +40,16 @@ pub enum PositionalType {
 }
 
 impl PositionalType {
-    crate fn name(&self) -> String {
+    crate fn to_coerce_hint(&self) -> Option<ExpressionKindHint> {
         match self {
-            PositionalType::Value(s) => s.clone(),
+            PositionalType::Value(_) => None,
-            PositionalType::Block(s) => s.clone(),
+            PositionalType::Block(_) => Some(ExpressionKindHint::Block),
-        }
-    }
-
-    crate fn evaluate(
-        &self,
-        arg: ast::Expression,
-        scope: &Scope,
-    ) -> Result<Spanned<Value>, ShellError> {
-        match self {
-            PositionalType::Value(_) => evaluate_expr(&arg, scope),
-            PositionalType::Block(_) => match arg {
-                ast::Expression {
-                    expr: ast::RawExpression::Block(b),
-                    ..
-                } => Ok(Spanned::from_item(Value::block(b.expr), arg.span.clone())),
-                ast::Expression {
-                    expr: ast::RawExpression::Binary(binary),
-                    ..
-                } => {
-                    // TODO: Use original spans
-                    let mut b = ast::ExpressionBuilder::new();
-                    if let Some(s) = binary.left.as_string() {
-                        Ok(Spanned::from_item(
-                            Value::block(b.binary((
-                                &|b| b.path((&|b| b.var("it"), vec![s.clone()])),
-                                &|_| binary.operator.clone(),
-                                &|_| binary.right.clone(),
-                            ))),
-                            arg.span.clone(),
-                        ))
-                    } else {
-                        let mut b = ast::ExpressionBuilder::new();
-                        let expr = b.binary((
-                            &|_| binary.left.clone(),
-                            &|_| binary.operator.clone(),
-                            &|_| binary.right.clone(),
-                        ));
-
-                        Ok(Spanned::from_item(Value::block(expr), arg.span.clone()))
-                    }
-                }
-                other => {
-                    let span = other.span.clone();
-                    Ok(Spanned::from_item(Value::block(other), span))
-                }
-            },
         }
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Getters)]
+#[get = "crate"]
 pub struct CommandConfig {
     crate name: String,
     crate mandatory_positional: Vec<PositionalType>,
@@ -94,90 +58,218 @@ pub struct CommandConfig {
     crate named: IndexMap<String, NamedType>,
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, new)]
 pub struct Args {
-    pub positional: Vec<Spanned<Value>>,
+    pub positional: Option<Vec<Spanned<Value>>>,
-    pub named: IndexMap<String, Value>,
+    pub named: Option<IndexMap<String, Spanned<Value>>>,
+}
+
+#[derive(new)]
+pub struct DebugPositional<'a> {
+    positional: &'a Option<Vec<Spanned<Value>>>,
+}
+
+impl fmt::Debug for DebugPositional<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match &self.positional {
+            None => write!(f, "None"),
+            Some(positional) => f
+                .debug_list()
+                .entries(positional.iter().map(|p| p.item().debug()))
+                .finish(),
+        }
+    }
+}
+
+#[derive(new)]
+pub struct DebugNamed<'a> {
+    named: &'a Option<IndexMap<String, Spanned<Value>>>,
+}
+
+impl fmt::Debug for DebugNamed<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match &self.named {
+            None => write!(f, "None"),
+            Some(named) => f
+                .debug_map()
+                .entries(named.iter().map(|(k, v)| (k, v.item().debug())))
+                .finish(),
+        }
+    }
+}
+
+pub struct DebugArgs<'a> {
+    args: &'a Args,
+}
+
+impl fmt::Debug for DebugArgs<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let mut s = f.debug_struct("Args");
+
+        s.field("positional", &DebugPositional::new(&self.args.positional));
+        s.field("named", &DebugNamed::new(&self.args.named));
+
+        s.finish()
+    }
+}
+
+impl Args {
+    pub fn debug(&'a self) -> DebugArgs<'a> {
+        DebugArgs { args: self }
+    }
+
+    pub fn nth(&self, pos: usize) -> Option<&Spanned<Value>> {
+        match &self.positional {
+            None => None,
+            Some(array) => array.iter().nth(pos),
+        }
+    }
+
+    pub fn expect_nth(&self, pos: usize) -> Result<&Spanned<Value>, ShellError> {
+        match &self.positional {
+            None => Err(ShellError::unimplemented("Better error: expect_nth")),
+            Some(array) => match array.iter().nth(pos) {
+                None => Err(ShellError::unimplemented("Better error: expect_nth")),
+                Some(item) => Ok(item),
+            },
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        match &self.positional {
+            None => 0,
+            Some(array) => array.len(),
+        }
+    }
+
+    pub fn has(&self, name: &str) -> bool {
+        match &self.named {
+            None => false,
+            Some(named) => named.contains_key(name),
+        }
+    }
+
+    pub fn get(&self, name: &str) -> Option<&Spanned<Value>> {
+        match &self.named {
+            None => None,
+            Some(named) => named.get(name),
+        }
+    }
+
+    pub fn positional_iter(&'a self) -> PositionalIter<'a> {
+        match &self.positional {
+            None => PositionalIter::Empty,
+            Some(v) => {
+                let iter = v.iter();
+                PositionalIter::Array(iter)
+            }
+        }
+    }
+}
+
+pub enum PositionalIter<'a> {
+    Empty,
+    Array(std::slice::Iter<'a, Spanned<Value>>),
+}
+
+impl Iterator for PositionalIter<'a> {
+    type Item = &'a Spanned<Value>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self {
+            PositionalIter::Empty => None,
+            PositionalIter::Array(iter) => iter.next(),
+        }
+    }
 }
 
 impl CommandConfig {
     crate fn evaluate_args(
         &self,
-        args: impl Iterator<Item = &'expr ast::Expression>,
+        call: &Spanned<CallNode>,
+        registry: &dyn CommandRegistry,
         scope: &Scope,
+        source: &str,
     ) -> Result<Args, ShellError> {
-        let mut positional: Vec<Spanned<Value>> = vec![];
+        let args = parse_command(self, registry, call, source)?;
-        let mut named: IndexMap<String, Value> = IndexMap::default();
 
-        let mut args: Vec<ast::Expression> = args.cloned().collect();
+        trace!("parsed args: {:?}", args);
 
-        for (key, ty) in self.named.iter() {
+        evaluate_args(args, registry, scope, source)
-            let index = args.iter().position(|a| a.is_flag(&key));
 
-            match (index, ty) {
+        // let mut positional: Vec<Spanned<Value>> = vec![];
-                (Some(i), NamedType::Switch) => {
+        // let mut named: IndexMap<String, Value> = IndexMap::default();
-                    args.remove(i);
-                    named.insert(key.clone(), Value::boolean(true));
-                }
 
-                (None, NamedType::Switch) => {}
+        // let mut args: Vec<TokenNode> = args.cloned().collect();
 
-                (Some(i), NamedType::Optional(v)) => {
+        // for (key, ty) in self.named.iter() {
-                    args.remove(i);
+        //     let index = args.iter().position(|a| a.is_flag(&key, source));
-                    named.insert(key.clone(), extract_named(&mut args, i, v)?);
-                }
 
-                (None, NamedType::Optional(_)) => {}
+        //     match (index, ty) {
+        //         (Some(i), NamedType::Switch) => {
+        //             args.remove(i);
+        //             named.insert(key.clone(), Value::boolean(true));
+        //         }
 
-                (Some(i), NamedType::Mandatory(v)) => {
+        //         (None, NamedType::Switch) => {}
-                    args.remove(i);
-                    named.insert(key.clone(), extract_named(&mut args, i, v)?);
-                }
 
-                (None, NamedType::Mandatory(_)) => {
+        //         (Some(i), NamedType::Optional(v)) => {
-                    return Err(ShellError::string(&format!(
+        //             args.remove(i);
-                        "Expected mandatory argument {}, but it was missing",
+        //             named.insert(key.clone(), extract_named(&mut args, i, v)?);
-                        key
+        //         }
-                    )))
-                }
-            }
-        }
 
-        let mut args = args.into_iter();
+        //         (None, NamedType::Optional(_)) => {}
 
-        for param in &self.mandatory_positional {
+        //         (Some(i), NamedType::Mandatory(v)) => {
-            let arg = args.next();
+        //             args.remove(i);
+        //             named.insert(key.clone(), extract_named(&mut args, i, v)?);
+        //         }
 
-            let value = match arg {
+        //         (None, NamedType::Mandatory(_)) => {
-                None => {
+        //             return Err(ShellError::string(&format!(
-                    return Err(ShellError::string(format!(
+        //                 "Expected mandatory argument {}, but it was missing",
-                        "expected mandatory positional argument {}",
+        //                 key
-                        param.name()
+        //             )))
-                    )))
+        //         }
-                }
+        //     }
+        // }
 
-                Some(arg) => param.evaluate(arg.clone(), scope)?,
+        // let mut args = args.into_iter();
-            };
 
-            positional.push(value);
+        // for param in &self.mandatory_positional {
-        }
+        //     let arg = args.next();
 
-        if self.rest_positional {
+        //     let value = match arg {
-            let rest: Result<Vec<Spanned<Value>>, _> =
+        //         None => {
-                args.map(|i| evaluate_expr(&i, &Scope::empty())).collect();
+        //             return Err(ShellError::string(format!(
-            positional.extend(rest?);
+        //                 "expected mandatory positional argument {}",
-        } else {
+        //                 param.name()
-            let rest: Vec<ast::Expression> = args.collect();
+        //             )))
+        //         }
 
-            if rest.len() > 0 {
+        //         Some(arg) => param.evaluate(arg.clone(), scope, source)?,
-                return Err(ShellError::string(&format!(
+        //     };
-                    "Too many arguments, extras: {:?}",
-                    rest
-                )));
-            }
-        }
 
-        Ok(Args { positional, named })
+        //     positional.push(value);
+        // }
+
+        // if self.rest_positional {
+        //     let rest: Result<Vec<Spanned<Value>>, _> = args
+        //         .map(|i| evaluate_baseline_expr(&i, &Scope::empty(), source))
+        //         .collect();
+        //     positional.extend(rest?);
+        // } else {
+        //     let rest: Vec<TokenNode> = args.collect();
+
+        //     if rest.len() > 0 {
+        //         return Err(ShellError::string(&format!(
+        //             "Too many arguments, extras: {:?}",
+        //             rest
+        //         )));
+        //     }
+        // }
+
+        // Ok(Args { positional, named })
     }
 
     #[allow(unused)]
@@ -186,50 +278,64 @@ impl CommandConfig {
     }
 }
 
-fn extract_named(
+fn evaluate_args(
-    v: &mut Vec<ast::Expression>,
+    args: hir::Call,
-    position: usize,
+    registry: &dyn CommandRegistry,
-    ty: &NamedValue,
+    scope: &Scope,
-) -> Result<Value, ShellError> {
+    source: &str,
-    match ty {
+) -> Result<Args, ShellError> {
-        NamedValue::Single => {
+    let positional: Result<Option<Vec<_>>, _> = args
-            let expr = v.remove(position);
+        .positional()
-            expect_simple_expr(expr)
+        .as_ref()
-        }
+        .map(|p| {
+            p.iter()
+                .map(|e| evaluate_baseline_expr(e, &(), scope, source))
+                .collect()
+        })
+        .transpose();
 
-        NamedValue::Tuple => {
+    let positional = positional?;
-            let expr = v.remove(position);
-            let next = v.remove(position);
 
-            let list = vec![expect_simple_expr(expr)?, expect_simple_expr(next)?];
+    let named: Result<Option<IndexMap<String, Spanned<Value>>>, ShellError> = args
-            Ok(Value::List(list))
+        .named()
-        }
+        .as_ref()
+        .map(|n| {
+            let mut results = IndexMap::new();
 
-        other => Err(ShellError::string(&format!(
+            for (name, value) in n.named.iter() {
-            "Unimplemented named argument {:?}",
+                match value {
-            other
+                    hir::named::NamedValue::PresentSwitch(span) => {
-        ))),
+                        results.insert(
-    }
+                            name.clone(),
-}
+                            Spanned::from_item(Value::boolean(true), *span),
+                        );
+                    }
+                    hir::named::NamedValue::Value(expr) => {
+                        results.insert(
+                            name.clone(),
+                            evaluate_baseline_expr(expr, registry, scope, source)?,
+                        );
+                    }
 
-fn expect_simple_expr(expr: ast::Expression) -> Result<Value, ShellError> {
+                    _ => {}
-    match &*expr {
+                };
-        ast::RawExpression::Leaf(l) => Ok(match l {
+            }
-            ast::Leaf::Bare(s) => Value::string(s.to_string()),
-            ast::Leaf::String(s) => Value::string(s),
-            ast::Leaf::Boolean(b) => Value::boolean(*b),
-            ast::Leaf::Int(i) => Value::int(*i),
-            ast::Leaf::Unit(i, unit) => unit.compute(*i),
-        }),
 
-        // TODO: Diagnostic
+            Ok(results)
-        other => Err(ShellError::string(&format!(
+        })
-            "Expected a value, found {}",
+        .transpose();
-            other.print()
+
-        ))),
+    let named = named?;
-    }
+
+    Ok(Args::new(positional, named))
 }
 
 pub trait CommandRegistry {
-    fn get(&self, name: &str) -> CommandConfig;
+    fn get(&self, name: &str) -> Option<CommandConfig>;
+}
+
+impl CommandRegistry for () {
+    fn get(&self, _name: &str) -> Option<CommandConfig> {
+        None
+    }
 }

src/prelude.rs

@@ -5,7 +5,6 @@ crate use crate::env::host::handle_unexpected;
 crate use crate::env::{Environment, Host};
 crate use crate::errors::ShellError;
 crate use crate::object::Value;
-crate use crate::parser::ast;
 crate use crate::stream::{single_output, InputStream, OutputStream};
 crate use futures::{FutureExt, StreamExt};
 crate use std::collections::VecDeque;

src/shell/helper.rs

@@ -1,9 +1,8 @@
 use crate::shell::completer::NuCompleter;
 
-use crate::parser::lexer::SpannedToken;
+use crate::parser::nom_input;
 use crate::prelude::*;
 use ansi_term::Color;
-use log::trace;
 use rustyline::completion::{self, Completer, FilenameCompleter};
 use rustyline::error::ReadlineError;
 use rustyline::highlight::Highlighter;
@@ -47,7 +46,7 @@ impl Hinter for Helper {
 }
 
 impl Highlighter for Helper {
-    fn highlight_prompt<'b, 's: 'b, 'p:'b>(&'s self, prompt: &'p str, _: bool) -> Cow<'b, str> {
+    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&'s self, prompt: &'p str, _: bool) -> Cow<'b, str> {
         Owned("\x1b[32m".to_owned() + &prompt[0..prompt.len() - 2] + "\x1b[m> ")
     }
 
@@ -56,30 +55,39 @@ impl Highlighter for Helper {
     }
 
     fn highlight<'l>(&self, line: &'l str, _pos: usize) -> Cow<'l, str> {
-        let tokens = crate::parser::lexer::Lexer::new(line, true);
+        let tokens = crate::parser::pipeline(nom_input(line));
-        let tokens: Result<Vec<(usize, SpannedToken, usize)>, _> = tokens.collect();
 
         match tokens {
             Err(_) => Cow::Borrowed(line),
-            Ok(v) => {
+            Ok((_rest, v)) => {
                 let mut out = String::new();
-                let mut iter = v.iter();
+                let tokens = match v.as_pipeline() {
+                    Err(_) => return Cow::Borrowed(line),
+                    Ok(v) => v,
+                };
 
-                let mut state = State::Command;
+                let mut iter = tokens.into_iter();
+
+                match iter.next() {
+                    None => return Cow::Owned(out),
+                    Some(v) => out.push_str(v.span().slice(line)),
+                };
 
                 loop {
                     match iter.next() {
                         None => return Cow::Owned(out),
-                        Some((start, token, end)) => {
+                        Some(token) => {
-                            let (style, new_state) = token_style(&token, state);
+                            // let styled = token_style(&token, state);
 
-                            trace!("token={:?}", token);
+                            // trace!("token={:?}", token);
-                            trace!("style={:?}", style);
+                            // trace!("style={:?}", style);
-                            trace!("new_state={:?}", new_state);
+                            // trace!("new_state={:?}", new_state);
 
-                            state = new_state;
+                            // state = new_state;
-                            let slice = &line[*start..*end];
+                            // let slice = &line[*start..*end];
-                            let styled = style.paint(slice);
+                            // let styled = style.paint(slice);
+                            out.push_str("|");
+                            let styled = Color::Black.bold().paint(token.span().slice(line));
                             out.push_str(&styled.to_string());
                         }
                     }
@@ -93,41 +101,4 @@ impl Highlighter for Helper {
     }
 }
 
-#[derive(Debug)]
-enum State {
-    Command,
-    Flag,
-    Var,
-    Bare,
-    None,
-}
-
-fn token_style(
-    token: &crate::parser::lexer::SpannedToken,
-    state: State,
-) -> (ansi_term::Style, State) {
-    use crate::parser::lexer::Token::*;
-
-    match (state, &token.token) {
-        (State::Command, Bare) => (Color::Cyan.bold(), State::None),
-        (State::Command, Whitespace) => (Color::White.normal(), State::Command),
-
-        (State::Flag, Bare) => (Color::Black.bold(), State::None),
-
-        (State::Var, Variable) => (Color::Yellow.bold(), State::None),
-
-        (State::Bare, PathDot) => (Color::Green.normal(), State::Bare),
-        (State::Bare, Member) => (Color::Green.normal(), State::Bare),
-
-        (_, Dash) | (_, DashDash) => (Color::Black.bold(), State::Flag),
-        (_, Dollar) => (Color::Yellow.bold(), State::Var),
-        (_, Bare) => (Color::Green.normal(), State::Bare),
-        (_, Member) => (Color::Cyan.normal(), State::None),
-        (_, Num) => (Color::Purple.bold(), State::None),
-        (_, DQString) | (_, SQString) => (Color::Green.normal(), State::None),
-        (_, Pipe) => (Color::White.normal(), State::Command),
-        _ => (Color::White.normal(), State::None),
-    }
-}
-
 impl rustyline::Helper for Helper {}

tests/sort_by.txt

@@ -1,3 +1,3 @@
 cd tests
-open test.toml --raw | split-row "\n" | skip 1 | first 4 | split-column "=" | sort-by Column1 | skip 1 | first 1 | get Column1 | trim | echo $it
+open test.toml --raw | lines | skip 1 | first 4 | split-column "=" | sort-by Column1 | skip 1 | first 1 | get Column1 | trim | echo $it
 exit

tests/split.txt

@@ -1,3 +1,3 @@
 cd tests
-open test.toml --raw | split-row "\n" | skip 1 | first 1 | split-column "=" | get Column1 | trim | echo $it
+open test.toml --raw | lines | skip 1 | first 1 | split-column "=" | get Column1 | trim | echo $it
 exit