Support evaluating most expressions

Blocks, paths, and others

Plus a bunch of other infra improvements

Cargo.toml

@@ -1,5 +1,3 @@
-cargo-features = ["default-run"]
-
 [package]
 name = "nu"
 version = "0.1.2"

src/commands/plugin.rs

@@ -1,12 +1,9 @@
 use crate::errors::ShellError;
-use crate::parser::registry::{CommandConfig, PositionalType};
-use crate::parser::Spanned;
 use crate::prelude::*;
 use serde::{self, Deserialize, Serialize};
 use std::io::prelude::*;
 use std::io::BufReader;
-use std::io::{Read, Write};
+use std::io::Write;
-use subprocess::Exec;
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct JsonRpc<T> {
@@ -14,6 +11,7 @@ pub struct JsonRpc<T> {
     pub method: String,
     pub params: T,
 }
+
 impl<T> JsonRpc<T> {
     pub fn new<U: Into<String>>(method: U, params: T) -> Self {
         JsonRpc {
@@ -50,15 +48,16 @@ pub fn plugin(plugin_name: String, args: CommandArgs) -> Result<OutputStream, Sh
         .expect("Failed to spawn child process");
 
     {
-        let mut stdin = child.stdin.as_mut().expect("Failed to open stdin");
+        let stdin = child.stdin.as_mut().expect("Failed to open stdin");
-        let mut stdout = child.stdout.as_mut().expect("Failed to open stdout");
+        let stdout = child.stdout.as_mut().expect("Failed to open stdout");
 
-        let mut reader = BufReader::new(stdout);
+        let _reader = BufReader::new(stdout);
 
         let request = JsonRpc::new("init", args.clone());
         let request_raw = serde_json::to_string(&request).unwrap();
-        stdin.write(format!("{}\n", request_raw).as_bytes());
+        stdin.write(format!("{}\n", request_raw).as_bytes())?;
     }
+
     let mut eos = VecDeque::new();
     eos.push_back(Value::Primitive(Primitive::EndOfStream));
 
@@ -66,25 +65,25 @@ pub fn plugin(plugin_name: String, args: CommandArgs) -> Result<OutputStream, Sh
         .chain(eos)
         .map(move |v| match v {
             Value::Primitive(Primitive::EndOfStream) => {
-                let mut stdin = child.stdin.as_mut().expect("Failed to open stdin");
+                let stdin = child.stdin.as_mut().expect("Failed to open stdin");
-                let mut stdout = child.stdout.as_mut().expect("Failed to open stdout");
+                let stdout = child.stdout.as_mut().expect("Failed to open stdout");
 
-                let mut reader = BufReader::new(stdout);
+                let _ = BufReader::new(stdout);
                 let request: JsonRpc<std::vec::Vec<Value>> = JsonRpc::new("quit", vec![]);
                 let request_raw = serde_json::to_string(&request).unwrap();
-                stdin.write(format!("{}\n", request_raw).as_bytes());
+                let _ = stdin.write(format!("{}\n", request_raw).as_bytes()); // TODO: Handle error
 
                 VecDeque::new()
             }
             _ => {
-                let mut stdin = child.stdin.as_mut().expect("Failed to open stdin");
+                let stdin = child.stdin.as_mut().expect("Failed to open stdin");
-                let mut stdout = child.stdout.as_mut().expect("Failed to open stdout");
+                let stdout = child.stdout.as_mut().expect("Failed to open stdout");
 
                 let mut reader = BufReader::new(stdout);
 
                 let request = JsonRpc::new("filter", v);
                 let request_raw = serde_json::to_string(&request).unwrap();
-                stdin.write(format!("{}\n", request_raw).as_bytes());
+                let _ = stdin.write(format!("{}\n", request_raw).as_bytes()); // TODO: Handle error
 
                 let mut input = String::new();
                 match reader.read_line(&mut input) {
@@ -101,7 +100,7 @@ pub fn plugin(plugin_name: String, args: CommandArgs) -> Result<OutputStream, Sh
                             }
                         }
                     }
-                    Err(x) => {
+                    Err(_) => {
                         let mut result = VecDeque::new();
                         result.push_back(ReturnValue::Value(Value::Error(Box::new(
                             ShellError::string("Error while processing input"),

src/errors.rs

@@ -36,7 +36,10 @@ impl Description {
 #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)]
 pub enum ShellError {
     String(StringError),
-    TypeError(Spanned<String>),
+    TypeError {
+        expected: String,
+        actual: Spanned<Option<String>>,
+    },
     MissingProperty {
         subpath: Description,
         expr: Description,
@@ -49,6 +52,16 @@ pub enum ShellError {
 }
 
 impl ShellError {
+    crate fn type_error(
+        expected: impl Into<String>,
+        actual: Spanned<impl Into<String>>,
+    ) -> ShellError {
+        ShellError::TypeError {
+            expected: expected.into(),
+            actual: actual.map(|i| Some(i.into())),
+        }
+    }
+
     crate fn parse_error(
         error: nom::Err<(nom_locate::LocatedSpan<&str>, nom::error::ErrorKind)>,
     ) -> ShellError {
@@ -57,9 +70,8 @@ impl ShellError {
         match error {
             nom::Err::Incomplete(_) => unreachable!(),
             nom::Err::Failure(span) | nom::Err::Error(span) => {
-                let diagnostic =
+                let diagnostic = Diagnostic::new(Severity::Error, format!("Parse Error"))
-                    Diagnostic::new(Severity::Error, format!("Parse Error"))
+                    .with_label(Label::new_primary(Span::from(span.0)));
-                        .with_label(Label::new_primary(Span::from(span.0)));
 
                 ShellError::diagnostic(diagnostic)
                 // nom::Context::Code(span, kind) => {
@@ -69,21 +81,20 @@ impl ShellError {
 
                 //     ShellError::diagnostic(diagnostic)
                 // }
-            }
+            } // ParseError::UnrecognizedToken {
-            // ParseError::UnrecognizedToken {
+              //     token: (start, SpannedToken { token, .. }, end),
-            //     token: (start, SpannedToken { token, .. }, end),
+              //     expected,
-            //     expected,
+              // } => {
-            // } => {
+              //     let diagnostic = Diagnostic::new(
-            //     let diagnostic = Diagnostic::new(
+              //         Severity::Error,
-            //         Severity::Error,
+              //         format!("Unexpected {:?}, expected {:?}", token, expected),
-            //         format!("Unexpected {:?}, expected {:?}", token, expected),
+              //     )
-            //     )
+              //     .with_label(Label::new_primary(Span::from((start, end))));
-            //     .with_label(Label::new_primary(Span::from((start, end))));
 
-            //     ShellError::diagnostic(diagnostic)
+              //     ShellError::diagnostic(diagnostic)
-            // }
+              // }
-            // ParseError::User { error } => error,
+              // ParseError::User { error } => error,
-            // other => ShellError::string(format!("{:?}", other)),
+              // other => ShellError::string(format!("{:?}", other)),
         }
     }
 
@@ -96,8 +107,23 @@ impl ShellError {
             ShellError::String(StringError { title, .. }) => {
                 Diagnostic::new(Severity::Error, title)
             }
-            ShellError::TypeError(s) => Diagnostic::new(Severity::Error, "Type Error")
+            ShellError::TypeError {
-                .with_label(Label::new_primary(s.span).with_message(s.item)),
+                expected,
+                actual:
+                    Spanned {
+                        item: Some(actual),
+                        span,
+                    },
+            } => Diagnostic::new(Severity::Error, "Type Error").with_label(
+                Label::new_primary(span)
+                    .with_message(format!("Expected {}, found {}", expected, actual)),
+            ),
+
+            ShellError::TypeError {
+                expected,
+                actual: Spanned { item: None, span },
+            } => Diagnostic::new(Severity::Error, "Type Error")
+                .with_label(Label::new_primary(span).with_message(expected)),
 
             ShellError::MissingProperty { subpath, expr } => {
                 let subpath = subpath.into_label();

src/evaluate/evaluator.rs

@@ -46,8 +46,8 @@ crate fn evaluate_baseline_expr(
             }
         }
         RawExpression::Block(block) => Ok(Spanned::from_item(
-            Value::Block(Block::new(*block.clone(), source.clone())),
+            Value::Block(Block::new(block.clone(), source.clone(), *expr.span())),
-            block.span(),
+            expr.span(),
         )),
         RawExpression::Path(path) => {
             let value = evaluate_baseline_expr(path.head(), registry, scope, source)?;

src/lib.rs

@@ -3,6 +3,7 @@
 #![feature(async_await)]
 #![feature(try_trait)]
 #![feature(bind_by_move_pattern_guards)]
+#![feature(box_syntax)]
 
 mod cli;
 mod commands;
@@ -19,6 +20,7 @@ mod shell;
 mod stream;
 
 pub use crate::commands::command::ReturnValue;
+pub use crate::parser::parse::span::SpannedItem;
 pub use crate::parser::Spanned;
 pub use cli::cli;
 pub use errors::ShellError;

src/object/base.rs

@@ -1,7 +1,7 @@
 use crate::errors::ShellError;
 use crate::evaluate::{evaluate_baseline_expr, Scope};
 use crate::object::DataDescriptor;
-use crate::parser::{hir, Operator, Spanned};
+use crate::parser::{hir, Operator, Span, Spanned};
 use crate::prelude::*;
 use crate::Text;
 use ansi_term::Color;
@@ -9,7 +9,7 @@ use chrono::{DateTime, Utc};
 use chrono_humanize::Humanize;
 use derive_new::new;
 use ordered_float::OrderedFloat;
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize, Serializer};
 use std::fmt;
 use std::time::SystemTime;
 
@@ -137,8 +137,9 @@ pub struct Operation {
 
 #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new)]
 pub struct Block {
-    crate expression: hir::Expression,
+    crate expressions: Vec<hir::Expression>,
     crate source: Text,
+    crate span: Span,
 }
 
 impl Serialize for Block {
@@ -146,7 +147,18 @@ impl Serialize for Block {
     where
         S: Serializer,
     {
-        serializer.serialize_str(&self.expression.source(&self.source.clone()))
+        let mut seq = serializer.serialize_seq(None)?;
+
+        let list = self
+            .expressions
+            .iter()
+            .map(|e| e.source(&self.source.clone()));
+
+        for item in list {
+            seq.serialize_element(item.as_ref())?;
+        }
+
+        seq.end()
     }
 }
 
@@ -167,7 +179,18 @@ impl Deserialize<'de> for Block {
 impl Block {
     pub fn invoke(&self, value: &Value) -> Result<Spanned<Value>, ShellError> {
         let scope = Scope::new(value.copy());
-        evaluate_baseline_expr(&self.expression, &(), &scope, &self.source)
+
+        if self.expressions.len() == 0 {
+            return Ok(Spanned::from_item(Value::nothing(), self.span));
+        }
+
+        let mut last = None;
+
+        for expr in self.expressions.iter() {
+            last = Some(evaluate_baseline_expr(&expr, &(), &scope, &self.source)?)
+        }
+
+        Ok(last.unwrap())
     }
 }
 
@@ -305,7 +328,12 @@ impl Value {
     crate fn format_leaf(&self, desc: Option<&DataDescriptor>) -> String {
         match self {
             Value::Primitive(p) => p.format(desc),
-            Value::Block(b) => b.expression.source(&b.source).to_string(),
+            Value::Block(b) => itertools::join(
+                b.expressions
+                    .iter()
+                    .map(|e| e.source(&b.source).to_string()),
+                "; ",
+            ),
             Value::Object(_) => format!("[object Object]"),
             Value::List(_) => format!("[list List]"),
             Value::Error(e) => format!("{}", e),
@@ -403,11 +431,6 @@ impl Value {
         }
     }
 
-    #[allow(unused)]
-    pub fn block(e: hir::Expression, source: Text) -> Value {
-        Value::Block(Block::new(e, source))
-    }
-
     pub fn string(s: impl Into<String>) -> Value {
         Value::Primitive(Primitive::String(s.into()))
     }

src/parser.rs

@@ -5,7 +5,7 @@ crate mod registry;
 
 use crate::errors::ShellError;
 
-crate use hir::baseline_parse_tokens::baseline_parse_tokens;
+crate use hir::baseline_parse_tokens::{baseline_parse_tokens, trace_remaining};
 crate use parse::call_node::CallNode;
 crate use parse::files::Files;
 crate use parse::flag::Flag;
@@ -14,7 +14,7 @@ crate use parse::parser::{nom_input, pipeline};
 crate use parse::pipeline::{Pipeline, PipelineElement};
 pub use parse::span::{Span, Spanned};
 crate use parse::text::Text;
-crate use parse::token_tree::TokenNode;
+crate use parse::token_tree::{DelimitedNode, Delimiter, PathNode, TokenNode};
 crate use parse::tokens::{RawToken, Token};
 crate use parse::unit::Unit;
 crate use parse_command::parse_command;

src/parser/hir.rs

@@ -9,11 +9,20 @@ use derive_new::new;
 use getset::Getters;
 
 crate use baseline_parse::baseline_parse_single_token;
-crate use baseline_parse_tokens::{baseline_parse_next_expr, ExpressionKindHint};
+crate use baseline_parse_tokens::{baseline_parse_next_expr, ExpressionKindHint, TokensIterator};
 crate use binary::Binary;
 crate use named::NamedArguments;
 crate use path::Path;
 
+pub fn path(head: impl Into<Expression>, tail: Vec<Spanned<impl Into<String>>>) -> Path {
+    Path::new(
+        head.into(),
+        tail.into_iter()
+            .map(|item| item.map(|string| string.into()))
+            .collect(),
+    )
+}
+
 #[derive(Debug, Clone, Eq, PartialEq, Getters, new)]
 pub struct Call {
     #[get = "crate"]
@@ -29,7 +38,7 @@ pub enum RawExpression {
     Literal(Literal),
     Variable(Variable),
     Binary(Box<Binary>),
-    Block(Box<Expression>),
+    Block(Vec<Expression>),
     Path(Box<Path>),
 
     #[allow(unused)]
@@ -76,6 +85,12 @@ impl Expression {
     }
 }
 
+impl From<Spanned<Path>> for Expression {
+    fn from(path: Spanned<Path>) -> Expression {
+        path.map(|p| RawExpression::Path(Box::new(p)))
+    }
+}
+
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
 pub enum Literal {
     Integer(i64),

src/parser/hir/baseline_parse_tokens.rs

@@ -1,24 +1,27 @@
 use crate::errors::ShellError;
 use crate::parser::registry::CommandRegistry;
-use crate::parser::{hir, hir::baseline_parse_single_token, Span, Spanned, TokenNode};
+use crate::parser::{
-use crate::Text;
+    hir, hir::baseline_parse_single_token, DelimitedNode, Delimiter, PathNode, RawToken, Span,
+    Spanned, TokenNode,
+};
+use crate::{SpannedItem, Text};
+use derive_new::new;
+use log::trace;
 
 pub fn baseline_parse_tokens(
-    token_nodes: &[TokenNode],
+    token_nodes: &mut TokensIterator<'_>,
     registry: &dyn CommandRegistry,
     source: &Text,
 ) -> Result<Vec<hir::Expression>, ShellError> {
     let mut exprs: Vec<hir::Expression> = vec![];
-    let mut rest = token_nodes;
 
     loop {
-        if rest.len() == 0 {
+        if token_nodes.at_end() {
             break;
         }
 
-        let (expr, remainder) = baseline_parse_next_expr(rest, registry, source, None)?;
+        let expr = baseline_parse_next_expr(token_nodes, registry, source, None)?;
         exprs.push(expr);
-        rest = remainder;
     }
 
     Ok(exprs)
@@ -35,25 +38,21 @@ pub enum ExpressionKindHint {
 }
 
 pub fn baseline_parse_next_expr(
-    token_nodes: &'nodes [TokenNode],
+    tokens: &mut TokensIterator,
-    _registry: &dyn CommandRegistry,
+    registry: &dyn CommandRegistry,
     source: &Text,
     coerce_hint: Option<ExpressionKindHint>,
-) -> Result<(hir::Expression, &'nodes [TokenNode]), ShellError> {
+) -> Result<hir::Expression, ShellError> {
-    let mut tokens = token_nodes.iter().peekable();
+    let first = match tokens.next() {
-
-    let first = next_token(&mut tokens);
-
-    let first = match first {
         None => return Err(ShellError::string("Expected token, found none")),
-        Some(token) => baseline_parse_semantic_token(token, source)?,
+        Some(token) => baseline_parse_semantic_token(token, registry, source)?,
     };
 
     let possible_op = tokens.peek();
 
     let op = match possible_op {
-        Some(TokenNode::Operator(op)) => op,
+        Some(TokenNode::Operator(op)) => op.clone(),
-        _ => return Ok((first, &token_nodes[1..])),
+        _ => return Ok(first),
     };
 
     tokens.next();
@@ -64,7 +63,7 @@ pub fn baseline_parse_next_expr(
                 "Expected op followed by another expr, found nothing",
             ))
         }
-        Some(token) => baseline_parse_semantic_token(token, source)?,
+        Some(token) => baseline_parse_semantic_token(token, registry, source)?,
     };
 
     // We definitely have a binary expression here -- let's see if we should coerce it into a block
@@ -72,11 +71,11 @@ pub fn baseline_parse_next_expr(
     match coerce_hint {
         None => {
             let span = (first.span.start, second.span.end);
-            let binary = hir::Binary::new(first, *op, second);
+            let binary = hir::Binary::new(first, op, second);
             let binary = hir::RawExpression::Binary(Box::new(binary));
             let binary = Spanned::from_item(binary, span);
 
-            Ok((binary, &token_nodes[3..]))
+            Ok(binary)
         }
 
         Some(hint) => match hint {
@@ -133,14 +132,14 @@ pub fn baseline_parse_next_expr(
                     }
                 };
 
-                let binary = hir::Binary::new(path, *op, second);
+                let binary = hir::Binary::new(path, op, second);
                 let binary = hir::RawExpression::Binary(Box::new(binary));
                 let binary = Spanned::from_item(binary, span);
 
-                let block = hir::RawExpression::Block(Box::new(binary));
+                let block = hir::RawExpression::Block(vec![binary]);
                 let block = Spanned::from_item(block, span);
 
-                Ok((block, &token_nodes[3..]))
+                Ok(block)
             }
 
             other => unimplemented!("coerce hint {:?}", other),
@@ -150,27 +149,199 @@ pub fn baseline_parse_next_expr(
 
 pub fn baseline_parse_semantic_token(
     token: &TokenNode,
+    registry: &dyn CommandRegistry,
     source: &Text,
 ) -> Result<hir::Expression, ShellError> {
     match token {
         TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)),
         TokenNode::Call(_call) => unimplemented!(),
-        TokenNode::Delimited(_delimited) => unimplemented!(),
+        TokenNode::Delimited(delimited) => baseline_parse_delimited(delimited, registry, source),
         TokenNode::Pipeline(_pipeline) => unimplemented!(),
         TokenNode::Operator(_op) => unreachable!(),
         TokenNode::Flag(_flag) => unimplemented!(),
         TokenNode::Identifier(_span) => unreachable!(),
         TokenNode::Whitespace(_span) => unreachable!(),
         TokenNode::Error(error) => Err(*error.item.clone()),
-        TokenNode::Path(_path) => unimplemented!(),
+        TokenNode::Path(path) => baseline_parse_path(path, registry, source),
     }
 }
 
-fn next_token(nodes: &mut impl Iterator<Item = &'a TokenNode>) -> Option<&'a TokenNode> {
+pub fn baseline_parse_delimited(
-    loop {
+    token: &Spanned<DelimitedNode>,
-        match nodes.next() {
+    registry: &dyn CommandRegistry,
-            Some(TokenNode::Whitespace(_)) => continue,
+    source: &Text,
-            other => return other,
+) -> Result<hir::Expression, ShellError> {
+    match token.delimiter() {
+        Delimiter::Brace => {
+            let children = token.children();
+            let exprs =
+                baseline_parse_tokens(&mut TokensIterator::new(children), registry, source)?;
+
+            let expr = hir::RawExpression::Block(exprs);
+            Ok(Spanned::from_item(expr, token.span()))
+        }
+        Delimiter::Paren => unimplemented!(),
+        Delimiter::Square => unimplemented!(),
+    }
+}
+
+pub fn baseline_parse_path(
+    token: &Spanned<PathNode>,
+    registry: &dyn CommandRegistry,
+    source: &Text,
+) -> Result<hir::Expression, ShellError> {
+    let head = baseline_parse_semantic_token(token.head(), registry, source)?;
+
+    let mut tail = vec![];
+
+    for part in token.tail() {
+        let string = match part {
+            TokenNode::Token(token) => match token.item() {
+                RawToken::Bare => token.span().slice(source),
+                RawToken::String(span) => span.slice(source),
+                RawToken::Integer(_) | RawToken::Size(..) | RawToken::Variable(_) => {
+                    return Err(ShellError::type_error(
+                        "String",
+                        token.type_name().spanned(part),
+                    ))
+                }
+            },
+
+            TokenNode::Identifier(span) => span.slice(source),
+
+            // TODO: Make this impossible
+            other => unreachable!("{:?}", other),
+        }
+        .to_string();
+
+        tail.push(string.spanned(part));
+    }
+
+    Ok(hir::path(head, tail).spanned(token).into())
+}
+
+#[derive(Debug, new)]
+pub struct TokensIterator<'a> {
+    tokens: &'a [TokenNode],
+    #[new(default)]
+    index: usize,
+    #[new(default)]
+    seen: indexmap::IndexSet<usize>,
+}
+
+impl TokensIterator<'a> {
+    pub fn remove(&mut self, position: usize) {
+        self.seen.insert(position);
+    }
+
+    pub fn len(&self) -> usize {
+        self.tokens.len()
+    }
+
+    pub fn at_end(&self) -> bool {
+        for index in self.index..self.tokens.len() {
+            if !self.seen.contains(&index) {
+                return false;
+            }
+        }
+
+        true
+    }
+
+    pub fn advance(&mut self) {
+        self.seen.insert(self.index);
+        self.index += 1;
+    }
+
+    pub fn extract<T>(&mut self, f: impl Fn(&TokenNode) -> Option<T>) -> Option<(usize, T)> {
+        for (i, item) in self.tokens.iter().enumerate() {
+            if self.seen.contains(&i) {
+                continue;
+            }
+
+            match f(item) {
+                None => {
+                    continue;
+                }
+                Some(value) => {
+                    self.seen.insert(i);
+                    return Some((i, value));
+                }
+            }
+        }
+
+        None
+    }
+
+    pub fn move_to(&mut self, pos: usize) {
+        self.index = pos;
+    }
+
+    pub fn restart(&mut self) {
+        self.index = 0;
+    }
+
+    pub fn clone(&self) -> TokensIterator {
+        TokensIterator {
+            tokens: self.tokens,
+            index: self.index,
+            seen: self.seen.clone(),
+        }
+    }
+
+    pub fn peek(&self) -> Option<&TokenNode> {
+        let mut tokens = self.clone();
+
+        tokens.next()
+    }
+
+    pub fn debug_remaining(&self) -> Vec<TokenNode> {
+        let mut tokens = self.clone();
+        tokens.restart();
+        tokens.cloned().collect()
+    }
+}
+
+impl Iterator for TokensIterator<'a> {
+    type Item = &'a TokenNode;
+
+    fn next(&mut self) -> Option<&'a TokenNode> {
+        loop {
+            if self.index >= self.tokens.len() {
+                return None;
+            }
+
+            if self.seen.contains(&self.index) {
+                self.advance();
+                continue;
+            }
+
+            if self.index >= self.tokens.len() {
+                return None;
+            }
+
+            match &self.tokens[self.index] {
+                TokenNode::Whitespace(_) => {
+                    self.advance();
+                }
+                other => {
+                    self.advance();
+                    return Some(other);
+                }
+            }
         }
     }
 }
+
+pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'a>, source: &Text) {
+    trace!(
+        "{} = {:?}",
+        desc,
+        itertools::join(
+            tail.debug_remaining()
+                .iter()
+                .map(|i| format!("%{:?}%", i.debug(source))),
+            " "
+        )
+    );
+}

src/parser/parse/span.rs

@@ -12,6 +12,14 @@ pub struct Spanned<T> {
     pub item: T,
 }
 
+pub trait SpannedItem: Sized {
+    fn spanned(self, span: impl Into<Span>) -> Spanned<Self> {
+        Spanned::from_item(self, span.into())
+    }
+}
+
+impl<T> SpannedItem for T {}
+
 impl<T> std::ops::Deref for Spanned<T> {
     type Target = T;
 
@@ -56,6 +64,12 @@ pub struct Span {
     // source: &'source str,
 }
 
+impl<T> From<&Spanned<T>> for Span {
+    fn from(input: &Spanned<T>) -> Span {
+        input.span
+    }
+}
+
 impl From<&Span> for Span {
     fn from(input: &Span) -> Span {
         *input

src/parser/parse/token_tree.rs

@@ -3,6 +3,8 @@ use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, span
 use crate::Text;
 use derive_new::new;
 use enum_utils::FromStr;
+use getset::Getters;
+use std::fmt;
 
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
 pub enum TokenNode {
@@ -20,6 +22,67 @@ pub enum TokenNode {
     Path(Spanned<PathNode>),
 }
 
+pub struct DebugTokenNode<'a> {
+    node: &'a TokenNode,
+    source: &'a Text,
+}
+
+impl fmt::Debug for DebugTokenNode<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self.node {
+            TokenNode::Token(t) => write!(f, "{:?}", t.debug(self.source)),
+            TokenNode::Call(s) => {
+                write!(f, "(")?;
+
+                write!(f, "{:?}", s.head().debug(self.source))?;
+
+                if let Some(children) = s.children() {
+                    for child in children {
+                        write!(f, "{:?}", child.debug(self.source))?;
+                    }
+                }
+
+                write!(f, ")")
+            }
+
+            TokenNode::Delimited(d) => {
+                write!(
+                    f,
+                    "{}",
+                    match d.delimiter {
+                        Delimiter::Brace => "{",
+                        Delimiter::Paren => "(",
+                        Delimiter::Square => "[",
+                    }
+                )?;
+
+                for child in d.children() {
+                    write!(f, "{:?}", child.debug(self.source))?;
+                }
+
+                write!(
+                    f,
+                    "{}",
+                    match d.delimiter {
+                        Delimiter::Brace => "}",
+                        Delimiter::Paren => ")",
+                        Delimiter::Square => "]",
+                    }
+                )
+            }
+            TokenNode::Pipeline(s) => write!(f, "<todo:pipeline>"),
+            TokenNode::Error(s) => write!(f, "<error> for {:?}", s.span().slice(self.source)),
+            rest => write!(f, "{}", rest.span().slice(self.source)),
+        }
+    }
+}
+
+impl From<&TokenNode> for Span {
+    fn from(token: &TokenNode) -> Span {
+        token.span()
+    }
+}
+
 impl TokenNode {
     pub fn span(&self) -> Span {
         match self {
@@ -36,6 +99,10 @@ impl TokenNode {
         }
     }
 
+    pub fn debug(&'a self, source: &'a Text) -> DebugTokenNode<'a> {
+        DebugTokenNode { node: self, source }
+    }
+
     pub fn as_external_arg(&self, source: &Text) -> String {
         self.span().slice(source).to_string()
     }
@@ -73,7 +140,8 @@ impl TokenNode {
     }
 }
 
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)]
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
+#[get = "crate"]
 pub struct DelimitedNode {
     delimiter: Delimiter,
     children: Vec<TokenNode>,
@@ -86,7 +154,8 @@ pub enum Delimiter {
     Square,
 }
 
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)]
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
+#[get = "crate"]
 pub struct PathNode {
     head: Box<TokenNode>,
     tail: Vec<TokenNode>,

src/parser/parse/tokens.rs

@@ -1,5 +1,7 @@
 use crate::parser::parse::span::*;
 use crate::parser::parse::unit::*;
+use crate::Text;
+use std::fmt;
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
 pub enum RawToken {
@@ -10,4 +12,36 @@ pub enum RawToken {
     Bare,
 }
 
+impl RawToken {
+    pub fn type_name(&self) -> &'static str {
+        match self {
+            RawToken::Integer(_) => "Integer",
+            RawToken::Size(..) => "Size",
+            RawToken::String(_) => "String",
+            RawToken::Variable(_) => "Variable",
+            RawToken::Bare => "String",
+        }
+    }
+}
+
 pub type Token = Spanned<RawToken>;
+
+impl Token {
+    pub fn debug(&self, source: &'a Text) -> DebugToken<'a> {
+        DebugToken {
+            node: *self,
+            source,
+        }
+    }
+}
+
+pub struct DebugToken<'a> {
+    node: Token,
+    source: &'a Text,
+}
+
+impl fmt::Debug for DebugToken<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.node.span().slice(self.source))
+    }
+}

src/parser/parse_command.rs

@@ -67,77 +67,81 @@ fn parse_command_tail(
     tail: Option<Vec<TokenNode>>,
     source: &Text,
 ) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ShellError> {
-    let mut tail = match tail {
+    let tail = &mut match &tail {
         None => return Ok(None),
-        Some(tail) => tail,
+        Some(tail) => hir::TokensIterator::new(tail),
     };
 
     let mut named = NamedArguments::new();
 
+    trace_remaining("nodes", tail.clone(), source);
+
     for (name, kind) in config.named() {
         trace!("looking for {} : {:?}", name, kind);
 
         match kind {
             NamedType::Switch => {
-                let (rest, flag) = extract_switch(name, tail, source);
+                let flag = extract_switch(name, tail, source);
-
-                tail = rest;
 
                 named.insert_switch(name, flag);
             }
             NamedType::Mandatory(kind) => match extract_mandatory(name, tail, source) {
                 Err(err) => return Err(err), // produce a correct diagnostic
-                Ok((rest, pos, _flag)) => {
+                Ok((pos, _flag)) => {
-                    let (expr, rest) = hir::baseline_parse_next_expr(
+                    tail.move_to(pos);
-                        &rest[pos..],
+                    let expr = hir::baseline_parse_next_expr(
+                        tail,
                         registry,
                         source,
                         kind.to_coerce_hint(),
                     )?;
-                    tail = rest.to_vec();
 
+                    tail.restart();
                     named.insert_mandatory(name, expr);
                 }
             },
             NamedType::Optional(kind) => match extract_optional(name, tail, source) {
                 Err(err) => return Err(err), // produce a correct diagnostic
-                Ok((rest, Some((pos, _flag)))) => {
+                Ok(Some((pos, _flag))) => {
-                    let (expr, rest) = hir::baseline_parse_next_expr(
+                    tail.move_to(pos);
-                        &rest[pos..],
+                    let expr = hir::baseline_parse_next_expr(
+                        tail,
                         registry,
                         source,
                         kind.to_coerce_hint(),
                     )?;
-                    tail = rest.to_vec();
 
+                    tail.restart();
                     named.insert_optional(name, Some(expr));
                 }
 
-                Ok((rest, None)) => {
+                Ok(None) => {
-                    tail = rest;
+                    tail.restart();
-
                     named.insert_optional(name, None);
                 }
             },
         };
     }
 
+    trace_remaining("after named", tail.clone(), source);
+
     let mut positional = vec![];
     let mandatory = config.mandatory_positional();
 
     for arg in mandatory {
+        trace!("Processing mandatory {:?}", arg);
+
         if tail.len() == 0 {
             return Err(ShellError::unimplemented("Missing mandatory argument"));
         }
 
-        let (result, rest) =
+        let result = hir::baseline_parse_next_expr(tail, registry, source, arg.to_coerce_hint())?;
-            hir::baseline_parse_next_expr(&tail, registry, source, arg.to_coerce_hint())?;
 
         positional.push(result);
-
-        tail = rest.to_vec();
     }
 
+    trace_remaining("after mandatory", tail.clone(), source);
+
     let optional = config.optional_positional();
 
     for arg in optional {
@@ -145,18 +149,19 @@ fn parse_command_tail(
             break;
         }
 
-        let (result, rest) =
+        let result = hir::baseline_parse_next_expr(tail, registry, source, arg.to_coerce_hint())?;
-            hir::baseline_parse_next_expr(&tail, registry, source, arg.to_coerce_hint())?;
 
         positional.push(result);
-
-        tail = rest.to_vec();
     }
 
+    trace_remaining("after optional", tail.clone(), source);
+
     // TODO: Only do this if rest params are specified
-    let remainder = baseline_parse_tokens(&tail, registry, source)?;
+    let remainder = baseline_parse_tokens(tail, registry, source)?;
     positional.extend(remainder);
 
+    trace_remaining("after rest", tail.clone(), source);
+
     trace!("Constructed positional={:?} named={:?}", positional, named);
 
     let positional = match positional {
@@ -174,38 +179,20 @@ fn parse_command_tail(
     Ok(Some((positional, named)))
 }
 
-fn extract_switch(
+fn extract_switch(name: &str, tokens: &mut hir::TokensIterator<'_>, source: &Text) -> Option<Flag> {
-    name: &str,
+    tokens
-    mut tokens: Vec<TokenNode>,
+        .extract(|t| t.as_flag(name, source))
-    source: &Text,
+        .map(|(_pos, flag)| flag.item)
-) -> (Vec<TokenNode>, Option<Flag>) {
-    let pos = tokens
-        .iter()
-        .enumerate()
-        .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f)))
-        .nth(0);
-
-    match pos {
-        None => (tokens, None),
-        Some((pos, flag)) => {
-            tokens.remove(pos);
-            (tokens, Some(*flag))
-        }
-    }
 }
 
 fn extract_mandatory(
     name: &str,
-    mut tokens: Vec<TokenNode>,
+    tokens: &mut hir::TokensIterator<'a>,
     source: &Text,
-) -> Result<(Vec<TokenNode>, usize, Flag), ShellError> {
+) -> Result<(usize, Flag), ShellError> {
-    let pos = tokens
+    let flag = tokens.extract(|t| t.as_flag(name, source));
-        .iter()
-        .enumerate()
-        .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f)))
-        .nth(0);
 
-    match pos {
+    match flag {
         None => Err(ShellError::unimplemented(
             "Better error: mandatory flags must be present",
         )),
@@ -218,24 +205,20 @@ fn extract_mandatory(
 
             tokens.remove(pos);
 
-            Ok((tokens, pos, *flag))
+            Ok((pos, *flag))
         }
     }
 }
 
 fn extract_optional(
     name: &str,
-    mut tokens: Vec<TokenNode>,
+    tokens: &mut hir::TokensIterator<'a>,
     source: &Text,
-) -> Result<(Vec<TokenNode>, Option<(usize, Flag)>), ShellError> {
+) -> Result<(Option<(usize, Flag)>), ShellError> {
-    let pos = tokens
+    let flag = tokens.extract(|t| t.as_flag(name, source));
-        .iter()
-        .enumerate()
-        .filter_map(|(i, t)| t.as_flag(name, source).map(|f| (i, f)))
-        .nth(0);
 
-    match pos {
+    match flag {
-        None => Ok((tokens, None)),
+        None => Ok(None),
         Some((pos, flag)) => {
             if tokens.len() <= pos {
                 return Err(ShellError::unimplemented(
@@ -245,7 +228,20 @@ fn extract_optional(
 
             tokens.remove(pos);
 
-            Ok((tokens, Some((pos, *flag))))
+            Ok(Some((pos, *flag)))
         }
     }
 }
+
+pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'a>, source: &Text) {
+    trace!(
+        "{} = {:?}",
+        desc,
+        itertools::join(
+            tail.debug_remaining()
+                .iter()
+                .map(|i| format!("%{:?}%", i.debug(source))),
+            " "
+        )
+    );
+}