Merge branch 'nom2' into nom4

src/cli.rs

@@ -260,7 +260,7 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
             debug!("=== Parsed ===");
             debug!("{:#?}", result);
 
-            let mut pipeline = classify_pipeline(&result, ctx, &line)?;
+            let mut pipeline = classify_pipeline(&result, ctx, &Text::from(line))?;
 
             match pipeline.commands.last() {
                 Some(ClassifiedCommand::Sink(_)) => {}
@@ -375,13 +375,13 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
 fn classify_pipeline(
     pipeline: &TokenNode,
     context: &Context,
-    source: &str,
+    source: &Text,
 ) -> Result<ClassifiedPipeline, ShellError> {
     let pipeline = pipeline.as_pipeline()?;
 
     let commands: Result<Vec<_>, ShellError> = pipeline
         .iter()
-        .map(|item| classify_command(&item, context, source))
+        .map(|item| classify_command(&item, context, &source))
         .collect();
 
     Ok(ClassifiedPipeline {
@@ -392,7 +392,7 @@ fn classify_pipeline(
 fn classify_command(
     command: &PipelineElement,
     context: &Context,
-    source: &str,
+    source: &Text,
 ) -> Result<ClassifiedCommand, ShellError> {
     let call = command.call();
 

src/commands/cd.rs

@@ -23,8 +23,8 @@ pub fn cd(args: CommandArgs) -> Result<OutputStream, ShellError> {
                     }
                 },
                 Some(v) => {
-                    let target = v.as_string()?.clone();
-                    match dunce::canonicalize(cwd.join(&target).as_path()) {
+                    let target = v.as_string()?;
+                    match dunce::canonicalize(cwd.join(target).as_path()) {
                         Ok(p) => p,
                         Err(_) => {
                             return Err(ShellError::labeled_error(

src/commands/classified.rs

@@ -126,7 +126,6 @@ impl InternalCommand {
 
         let mut result =
             context.run_command(self.command, self.name_span.clone(), self.args, objects)?;
-        let env = context.env.clone();
 
         let mut stream = VecDeque::new();
         while let Some(item) = result.next().await {

src/commands/config.rs

@@ -57,7 +57,7 @@ pub fn config(args: CommandArgs) -> Result<OutputStream, ShellError> {
 
     if let Some(v) = args.get("set") {
         if let Ok((key, value)) = v.as_pair() {
-            result.insert(key.as_string()?, value.clone());
+            result.insert(key.as_string()?.to_string(), value.clone());
 
             config::write_config(&result)?;
 

src/commands/enter.rs

@@ -1,17 +1,18 @@
 use crate::commands::command::CommandAction;
 use crate::errors::ShellError;
 use crate::object::{Primitive, Value};
-use crate::parser::Spanned;
 use crate::prelude::*;
 use std::path::{Path, PathBuf};
 
 pub fn enter(args: CommandArgs) -> Result<OutputStream, ShellError> {
     let path = match args.nth(0) {
-        None => return Err(ShellError::maybe_labeled_error(
+        None => {
+            return Err(ShellError::maybe_labeled_error(
                 "open requires a path or url",
                 "missing path",
                 args.name_span,
-        )),
+            ))
+        }
         Some(p) => p,
     };
 
@@ -64,7 +65,7 @@ pub fn enter(args: CommandArgs) -> Result<OutputStream, ShellError> {
                     }
                 }
             } else {
-                full_path.push(Path::new(&s));
+                full_path.push(Path::new(s));
                 match std::fs::read_to_string(&full_path) {
                     Ok(s) => (
                         full_path

src/commands/from_json.rs

@@ -4,12 +4,12 @@ use crate::prelude::*;
 
 fn convert_json_value_to_nu_value(v: &serde_hjson::Value) -> Value {
     match v {
-        serde_hjson::Value::Null => Value::Primitive(Primitive::String("".to_string())),
+        serde_hjson::Value::Null => Value::Primitive(Primitive::String(String::from(""))),
         serde_hjson::Value::Bool(b) => Value::Primitive(Primitive::Boolean(*b)),
         serde_hjson::Value::F64(n) => Value::Primitive(Primitive::Float(OF64::from(*n))),
         serde_hjson::Value::U64(n) => Value::Primitive(Primitive::Int(*n as i64)),
         serde_hjson::Value::I64(n) => Value::Primitive(Primitive::Int(*n as i64)),
-        serde_hjson::Value::String(s) => Value::Primitive(Primitive::String(s.clone())),
+        serde_hjson::Value::String(s) => Value::Primitive(Primitive::String(String::from(s))),
         serde_hjson::Value::Array(a) => Value::List(
             a.iter()
                 .map(|x| convert_json_value_to_nu_value(x))

src/commands/from_toml.rs

@@ -7,7 +7,7 @@ fn convert_toml_value_to_nu_value(v: &toml::Value) -> Value {
         toml::Value::Boolean(b) => Value::Primitive(Primitive::Boolean(*b)),
         toml::Value::Integer(n) => Value::Primitive(Primitive::Int(*n)),
         toml::Value::Float(n) => Value::Primitive(Primitive::Float(OF64::from(*n))),
-        toml::Value::String(s) => Value::Primitive(Primitive::String(s.clone())),
+        toml::Value::String(s) => Value::Primitive(Primitive::String(String::from(s))),
         toml::Value::Array(a) => Value::List(
             a.iter()
                 .map(|x| convert_toml_value_to_nu_value(x))

src/commands/from_xml.rs

@@ -32,13 +32,13 @@ fn from_node_to_value<'a, 'd>(n: &roxmltree::Node<'a, 'd>) -> Value {
 
         Value::Object(collected)
     } else if n.is_comment() {
-        Value::Primitive(Primitive::String("<comment>".to_string()))
+        Value::string("<comment>")
     } else if n.is_pi() {
-        Value::Primitive(Primitive::String("<processing_instruction>".to_string()))
+        Value::string("<processing_instruction>")
     } else if n.is_text() {
-        Value::Primitive(Primitive::String(n.text().unwrap().to_string()))
+        Value::string(n.text().unwrap())
     } else {
-        Value::Primitive(Primitive::String("<unknown>".to_string()))
+        Value::string("<unknown>")
     }
 }
 

src/commands/from_yaml.rs

@@ -11,7 +11,7 @@ fn convert_yaml_value_to_nu_value(v: &serde_yaml::Value) -> Value {
         serde_yaml::Value::Number(n) if n.is_f64() => {
             Value::Primitive(Primitive::Float(OF64::from(n.as_f64().unwrap())))
         }
-        serde_yaml::Value::String(s) => Value::Primitive(Primitive::String(s.clone())),
+        serde_yaml::Value::String(s) => Value::string(s),
         serde_yaml::Value::Sequence(a) => Value::List(
             a.iter()
                 .map(|x| convert_yaml_value_to_nu_value(x))

src/commands/get.rs

@@ -46,6 +46,7 @@ pub fn get(args: CommandArgs) -> Result<OutputStream, ShellError> {
         .positional_iter()
         .map(|a| (a.as_string().map(|x| (x, a.span))))
         .collect();
+
     let fields = fields?;
 
     let stream = args

src/commands/lines.rs

@@ -20,7 +20,7 @@ pub fn lines(args: CommandArgs) -> Result<OutputStream, ShellError> {
                 let mut result = VecDeque::new();
                 for s in split_result {
                     result.push_back(ReturnValue::Value(Value::Primitive(Primitive::String(
-                        s.to_string(),
+                        s.into(),
                     ))));
                 }
                 result

src/commands/ls.rs

@@ -14,7 +14,7 @@ pub fn ls(args: CommandArgs) -> Result<OutputStream, ShellError> {
         Some(Spanned {
             item: Value::Primitive(Primitive::String(s)),
             ..
-        }) => full_path.push(Path::new(s)),
+        }) => full_path.push(Path::new(&s)),
         _ => {}
     }
 

src/commands/open.rs

@@ -1,6 +1,5 @@
 use crate::errors::ShellError;
 use crate::object::{Primitive, Value};
-use crate::parser::parse2::span::Spanned;
 use crate::parser::registry::{CommandConfig, NamedType};
 use crate::prelude::*;
 use indexmap::IndexMap;
@@ -87,7 +86,7 @@ fn open(args: CommandArgs) -> Result<OutputStream, ShellError> {
                     }
                 }
             } else {
-                full_path.push(Path::new(&s));
+                full_path.push(Path::new(s));
                 match std::fs::read_to_string(&full_path) {
                     Ok(s) => (
                         full_path
@@ -222,9 +221,7 @@ fn open(args: CommandArgs) -> Result<OutputStream, ShellError> {
             ));
         }
         _ => {
-            stream.push_back(ReturnValue::Value(Value::Primitive(Primitive::String(
-                contents,
-            ))));
+            stream.push_back(ReturnValue::Value(Value::string(contents)));
         }
     }
 

src/commands/size.rs

@@ -63,7 +63,7 @@ fn count(name: &str, contents: &str) -> ReturnValue {
     }
 
     let mut dict = Dictionary::default();
-    dict.add("name", Value::string(name.to_owned()));
+    dict.add("name", Value::string(name));
     dict.add("lines", Value::int(lines));
     dict.add("words", Value::int(words));
     dict.add("chars", Value::int(chars));

src/commands/split_column.rs

@@ -36,19 +36,16 @@ pub fn split_column(args: CommandArgs) -> Result<OutputStream, ShellError> {
                     }
 
                     let mut dict = crate::object::Dictionary::default();
-                    for (k, v) in split_result.iter().zip(gen_columns.iter()) {
-                        dict.add(
-                            v.clone(),
-                            Value::Primitive(Primitive::String(k.to_string())),
-                        );
+                    for (&k, v) in split_result.iter().zip(gen_columns.iter()) {
+                        dict.add(v.clone(), Value::Primitive(Primitive::String(k.into())));
                     }
                     ReturnValue::Value(Value::Object(dict))
                 } else if split_result.len() == (positional.len() - 1) {
                     let mut dict = crate::object::Dictionary::default();
-                    for (k, v) in split_result.iter().zip(positional.iter().skip(1)) {
+                    for (&k, v) in split_result.iter().zip(positional.iter().skip(1)) {
                         dict.add(
                             v.as_string().unwrap(),
-                            Value::Primitive(Primitive::String(k.to_string())),
+                            Value::Primitive(Primitive::String(k.into())),
                         );
                     }
                     ReturnValue::Value(Value::Object(dict))
@@ -57,7 +54,7 @@ pub fn split_column(args: CommandArgs) -> Result<OutputStream, ShellError> {
                     for k in positional.iter().skip(1) {
                         dict.add(
                             k.as_string().unwrap().trim(),
-                            Value::Primitive(Primitive::String("".to_string())),
+                            Value::Primitive(Primitive::String("".into())),
                         );
                     }
                     ReturnValue::Value(Value::Object(dict))

src/commands/split_row.rs

@@ -32,7 +32,7 @@ pub fn split_row(args: CommandArgs) -> Result<OutputStream, ShellError> {
                 let mut result = VecDeque::new();
                 for s in split_result {
                     result.push_back(ReturnValue::Value(Value::Primitive(Primitive::String(
-                        s.to_string(),
+                        s.into(),
                     ))));
                 }
                 result

src/commands/trim.rs

@@ -11,7 +11,7 @@ pub fn trim(args: CommandArgs) -> Result<OutputStream, ShellError> {
     Ok(input
         .map(move |v| match v {
             Value::Primitive(Primitive::String(s)) => {
-                ReturnValue::Value(Value::Primitive(Primitive::String(s.trim().to_string())))
+                ReturnValue::Value(Value::Primitive(Primitive::String(s.trim().into())))
             }
             _ => ReturnValue::Value(Value::Error(Box::new(ShellError::maybe_labeled_error(
                 "Expected string values from pipeline",

src/commands/view.rs

@@ -42,7 +42,7 @@ pub fn view(args: CommandArgs) -> Result<OutputStream, ShellError> {
         .build()
         .map_err(|e| ShellError::string(e))?;
 
-    let file = cwd.join(target);
+    let file = cwd.join(&target);
 
     let _ = printer.file(file.display().to_string());
 

src/evaluate/evaluator.rs

@@ -1,7 +1,7 @@
 use crate::object::base::Block;
 use crate::parser::{
     hir::{self, Expression, RawExpression},
-    CommandRegistry, Span, Spanned, Text,
+    CommandRegistry, Spanned, Text,
 };
 use crate::prelude::*;
 use derive_new::new;
@@ -27,7 +27,7 @@ crate fn evaluate_baseline_expr(
     expr: &Expression,
     registry: &dyn CommandRegistry,
     scope: &Scope,
-    source: &str,
+    source: &Text,
 ) -> Result<Spanned<Value>, ShellError> {
     match &expr.item {
         RawExpression::Literal(literal) => Ok(evaluate_literal(expr.copy_span(*literal), source)),
@@ -45,7 +45,7 @@ crate fn evaluate_baseline_expr(
             }
         }
         RawExpression::Block(block) => Ok(Spanned::from_item(
-            Value::Block(Block::new(*block.clone(), Text::from(source))), // TODO: Pass Text around
+            Value::Block(Block::new(*block.clone(), source.clone())),
             block.span(),
         )),
         RawExpression::Path(path) => {
@@ -67,7 +67,7 @@ crate fn evaluate_baseline_expr(
     }
 }
 
-fn evaluate_literal(literal: Spanned<hir::Literal>, source: &str) -> Spanned<Value> {
+fn evaluate_literal(literal: Spanned<hir::Literal>, source: &Text) -> Spanned<Value> {
     let result = match literal.item {
         hir::Literal::Integer(int) => Value::int(int),
         hir::Literal::Size(_int, _unit) => unimplemented!(),
@@ -81,7 +81,7 @@ fn evaluate_literal(literal: Spanned<hir::Literal>, source: &str) -> Spanned<Val
 fn evaluate_reference(
     name: &hir::Variable,
     scope: &Scope,
-    source: &str,
+    source: &Text,
 ) -> Result<Spanned<Value>, ShellError> {
     match name {
         hir::Variable::It(span) => Ok(Spanned::from_item(scope.it.copy(), span)),

src/format/generic.rs

@@ -31,7 +31,7 @@ impl RenderView for GenericView<'value> {
 
             b @ Value::Block(_) => {
                 let printed = b.format_leaf(None);
-                let view = EntriesView::from_value(&Value::string(&printed));
+                let view = EntriesView::from_value(&Value::string(printed));
                 view.render_view(host)?;
                 Ok(())
             }

src/main.rs

@@ -22,6 +22,8 @@ use clap::{App, Arg};
 use log::LevelFilter;
 use std::error::Error;
 
+crate use parser::parse2::text::Text;
+
 fn main() -> Result<(), Box<dyn Error>> {
     let matches = App::new("nu shell")
         .version("0.5")

src/object/base.rs

@@ -3,17 +3,16 @@ use crate::evaluate::{evaluate_baseline_expr, Scope};
 use crate::object::DataDescriptor;
 use crate::parser::{hir, Operator, Spanned};
 use crate::prelude::*;
+use crate::Text;
 use ansi_term::Color;
 use chrono::{DateTime, Utc};
 use chrono_humanize::Humanize;
 use derive_new::new;
 use ordered_float::OrderedFloat;
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use std::fmt;
 use std::time::SystemTime;
 
-use crate::parser::Text;
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
-
 #[derive(Debug, Clone, Copy, Ord, PartialOrd, Eq, PartialEq, new)]
 pub struct OF64 {
     crate inner: OrderedFloat<f64>,
@@ -141,7 +140,7 @@ impl Serialize for Block {
     where
         S: Serializer,
     {
-        serializer.serialize_str(&self.expression.source(self.source.as_ref()))
+        serializer.serialize_str(&self.expression.source(&self.source.clone()))
     }
 }
 
@@ -162,7 +161,7 @@ impl Deserialize<'de> for Block {
 impl Block {
     pub fn invoke(&self, value: &Value) -> Result<Spanned<Value>, ShellError> {
         let scope = Scope::new(value.copy());
-        evaluate_baseline_expr(&self.expression, &(), &scope, self.source.as_ref())
+        evaluate_baseline_expr(&self.expression, &(), &scope, &self.source)
     }
 }
 
@@ -293,7 +292,7 @@ impl Value {
     crate fn format_leaf(&self, desc: Option<&DataDescriptor>) -> String {
         match self {
             Value::Primitive(p) => p.format(desc),
-            Value::Block(b) => b.expression.source(b.source.as_ref()).to_string(),
+            Value::Block(b) => b.expression.source(&b.source).to_string(),
             Value::Object(_) => format!("[object Object]"),
             Value::List(_) => format!("[list List]"),
             Value::Error(e) => format!("{}", e),
@@ -348,12 +347,11 @@ impl Value {
 
     crate fn as_string(&self) -> Result<String, ShellError> {
         match self {
-            Value::Primitive(Primitive::String(x)) => Ok(format!("{}", x)),
+            Value::Primitive(Primitive::String(s)) => Ok(s.clone()),
             Value::Primitive(Primitive::Boolean(x)) => Ok(format!("{}", x)),
             Value::Primitive(Primitive::Float(x)) => Ok(format!("{}", x.into_inner())),
             Value::Primitive(Primitive::Int(x)) => Ok(format!("{}", x)),
             Value::Primitive(Primitive::Bytes(x)) => Ok(format!("{}", x)),
-            //Value::Primitive(Primitive::String(s)) => Ok(s.clone()),
             // TODO: this should definitely be more general with better errors
             other => Err(ShellError::string(format!(
                 "Expected string, got {:?}",

src/object/desc.rs

@@ -1,4 +1,5 @@
 use crate::object::types::Type;
+use crate::Text;
 use derive_new::new;
 use serde::{Deserialize, Serialize, Serializer};
 
@@ -90,9 +91,19 @@ impl From<String> for DataDescriptor {
     }
 }
 
+impl From<Text> for DataDescriptor {
+    fn from(input: Text) -> DataDescriptor {
+        DataDescriptor {
+            name: DescriptorName::String(input.to_string()),
+            readonly: true,
+            ty: Type::Any,
+        }
+    }
+}
+
 impl DescriptorName {
-    crate fn for_string_name(name: impl Into<String>) -> DescriptorName {
-        DescriptorName::String(name.into())
+    crate fn for_string_name(name: impl AsRef<str>) -> DescriptorName {
+        DescriptorName::String(name.as_ref().into())
     }
 }
 
@@ -113,7 +124,7 @@ impl DataDescriptor {
         }
     }
 
-    crate fn for_string_name(name: impl Into<String>) -> DataDescriptor {
+    crate fn for_string_name(name: impl AsRef<str>) -> DataDescriptor {
         DataDescriptor::for_name(DescriptorName::for_string_name(name))
     }
 

src/parser/hir/baseline_parse.rs

@@ -1,6 +1,7 @@
 use crate::parser::{hir, RawToken, Token};
+use crate::Text;
 
-pub fn baseline_parse_single_token(token: &Token, source: &str) -> hir::Expression {
+pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Expression {
     match *token.item() {
         RawToken::Integer(int) => hir::Expression::int(int, token.span),
         RawToken::Size(int, unit) => hir::Expression::size(int, unit, token.span),

src/parser/hir/baseline_parse_tokens.rs

@@ -1,11 +1,12 @@
 use crate::errors::ShellError;
 use crate::parser::registry::CommandRegistry;
 use crate::parser::{hir, hir::baseline_parse_single_token, Span, Spanned, TokenNode};
+use crate::Text;
 
 pub fn baseline_parse_tokens(
     token_nodes: &[TokenNode],
     registry: &dyn CommandRegistry,
-    source: &str,
+    source: &Text,
 ) -> Result<Vec<hir::Expression>, ShellError> {
     let mut exprs: Vec<hir::Expression> = vec![];
     let mut rest = token_nodes;
@@ -36,7 +37,7 @@ pub enum ExpressionKindHint {
 pub fn baseline_parse_next_expr(
     token_nodes: &'nodes [TokenNode],
     _registry: &dyn CommandRegistry,
-    source: &str,
+    source: &Text,
     coerce_hint: Option<ExpressionKindHint>,
 ) -> Result<(hir::Expression, &'nodes [TokenNode]), ShellError> {
     let mut tokens = token_nodes.iter().peekable();
@@ -143,7 +144,7 @@ pub fn baseline_parse_next_expr(
 
 pub fn baseline_parse_semantic_token(
     token: &TokenNode,
-    source: &str,
+    source: &Text,
 ) -> Result<hir::Expression, ShellError> {
     match token {
         TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)),

src/parser/parse2/span.rs

@@ -1,3 +1,4 @@
+use crate::Text;
 use derive_new::new;
 use getset::Getters;
 
@@ -40,8 +41,8 @@ impl<T> Spanned<T> {
         }
     }
 
-    pub fn source(&self, source: &'source str) -> &'source str {
-        self.span().slice(source)
+    pub fn source(&self, source: &Text) -> Text {
+        Text::from(self.span().slice(source))
     }
 }
 
@@ -95,7 +96,7 @@ impl From<&std::ops::Range<usize>> for Span {
 }
 
 impl Span {
-    pub fn slice(&self, source: &'source str) -> &'source str {
+    pub fn slice(&self, source: &'a str) -> &'a str {
         &source[self.start..self.end]
     }
 }

src/parser/parse2/text.rs

@@ -31,7 +31,7 @@ impl Text {
     /// Extract a new `Text` that is a subset of an old `Text`
     /// -- `text.extract(1..3)` is similar to `&foo[1..3]` except that
     /// it gives back an owned value instead of a borrowed value.
-    pub fn extract(&self, range: Range<usize>) -> Self {
+    pub fn slice(&self, range: Range<usize>) -> Self {
         let mut result = self.clone();
         result.select(range);
         result

src/parser/parse2/token_tree.rs

@@ -1,5 +1,6 @@
 use crate::errors::ShellError;
 use crate::parser::parse2::{call_node::*, flag::*, operator::*, span::*, tokens::*};
+use crate::Text;
 use derive_new::new;
 use enum_utils::FromStr;
 use getset::Getters;
@@ -36,11 +37,11 @@ impl TokenNode {
         }
     }
 
-    pub fn as_external_arg(&self, source: &str) -> String {
+    pub fn as_external_arg(&self, source: &Text) -> String {
         self.span().slice(source).to_string()
     }
 
-    pub fn source(&self, source: &'source str) -> &'source str {
+    pub fn source(&self, source: &'a Text) -> &'a str {
         self.span().slice(source)
     }
 
@@ -54,7 +55,7 @@ impl TokenNode {
         }
     }
 
-    crate fn as_flag(&self, value: &str, source: &str) -> Option<Spanned<Flag>> {
+    crate fn as_flag(&self, value: &str, source: &Text) -> Option<Spanned<Flag>> {
         match self {
             TokenNode::Flag(
                 flag @ Spanned {
@@ -99,19 +100,3 @@ pub struct PipelineElement {
     call: Spanned<CallNode>,
     pub post_ws: Option<Span>,
 }
-
-impl PipelineElement {
-    crate fn span(&self) -> Span {
-        let start = match self.pre_ws {
-            None => self.call.span.start,
-            Some(span) => span.start,
-        };
-
-        let end = match self.post_ws {
-            None => self.call.span.end,
-            Some(span) => span.end,
-        };
-
-        Span::from((start, end))
-    }
-}

src/parser/parse2/tokens.rs

@@ -11,28 +11,3 @@ pub enum RawToken {
 }
 
 pub type Token = Spanned<RawToken>;
-
-impl Token {
-    pub fn to_semantic_token(&self) -> Option<SemanticToken> {
-        let semantic_token = match self.item {
-            RawToken::Integer(int) => RawSemanticToken::Integer(int),
-            RawToken::Size(int, unit) => RawSemanticToken::Size(int, unit),
-            RawToken::String(span) => RawSemanticToken::String(span),
-            RawToken::Variable(span) => RawSemanticToken::Variable(span),
-            RawToken::Bare => RawSemanticToken::Bare,
-        };
-
-        Some(Spanned::from_item(semantic_token, self.span))
-    }
-}
-
-#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
-pub enum RawSemanticToken {
-    Integer(i64),
-    Size(i64, Unit),
-    String(Span),
-    Variable(Span),
-    Bare,
-}
-
-pub type SemanticToken = Spanned<RawSemanticToken>;

src/parser/parse_command.rs

@@ -5,13 +5,14 @@ use crate::parser::{
     hir::{self, NamedArguments},
     Flag, RawToken, TokenNode,
 };
+use crate::Text;
 use log::trace;
 
 pub fn parse_command(
     config: &CommandConfig,
     registry: &dyn CommandRegistry,
     call: &Spanned<CallNode>,
-    source: &str,
+    source: &Text,
 ) -> Result<hir::Call, ShellError> {
     let Spanned { item: call, .. } = call;
 
@@ -64,7 +65,7 @@ fn parse_command_tail(
     config: &CommandConfig,
     registry: &dyn CommandRegistry,
     tail: Option<Vec<TokenNode>>,
-    source: &str,
+    source: &Text,
 ) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ShellError> {
     let mut tail = match tail {
         None => return Ok(None),
@@ -176,7 +177,7 @@ fn parse_command_tail(
 fn extract_switch(
     name: &str,
     mut tokens: Vec<TokenNode>,
-    source: &str,
+    source: &Text,
 ) -> (Vec<TokenNode>, Option<Flag>) {
     let pos = tokens
         .iter()
@@ -196,7 +197,7 @@ fn extract_switch(
 fn extract_mandatory(
     name: &str,
     mut tokens: Vec<TokenNode>,
-    source: &str,
+    source: &Text,
 ) -> Result<(Vec<TokenNode>, usize, Flag), ShellError> {
     let pos = tokens
         .iter()
@@ -225,7 +226,7 @@ fn extract_mandatory(
 fn extract_optional(
     name: &str,
     mut tokens: Vec<TokenNode>,
-    source: &str,
+    source: &Text,
 ) -> Result<(Vec<TokenNode>, Option<(usize, Flag)>), ShellError> {
     let pos = tokens
         .iter()

src/parser/registry.rs

@@ -189,7 +189,7 @@ impl CommandConfig {
         call: &Spanned<CallNode>,
         registry: &dyn CommandRegistry,
         scope: &Scope,
-        source: &str,
+        source: &Text,
     ) -> Result<Args, ShellError> {
         let args = parse_command(self, registry, call, source)?;
 
@@ -282,7 +282,7 @@ fn evaluate_args(
     args: hir::Call,
     registry: &dyn CommandRegistry,
     scope: &Scope,
-    source: &str,
+    source: &Text,
 ) -> Result<Args, ShellError> {
     let positional: Result<Option<Vec<_>>, _> = args
         .positional()

src/prelude.rs

@@ -6,6 +6,7 @@ crate use crate::env::{Environment, Host};
 crate use crate::errors::ShellError;
 crate use crate::object::Value;
 crate use crate::stream::{single_output, InputStream, OutputStream};
+crate use crate::Text;
 crate use futures::{FutureExt, StreamExt};
 crate use std::collections::VecDeque;
 crate use std::pin::Pin;

src/shell/helper.rs

@@ -1,10 +1,10 @@
-use crate::shell::completer::NuCompleter;
-use crate::parser::parse2::PipelineElement;
+use crate::parser::nom_input;
+use crate::parser::parse2::span::Spanned;
 use crate::parser::parse2::token_tree::TokenNode;
 use crate::parser::parse2::tokens::RawToken;
-use crate::parser::parse2::span::Spanned;
-use crate::parser::nom_input;
+use crate::parser::parse2::PipelineElement;
 use crate::prelude::*;
+use crate::shell::completer::NuCompleter;
 use ansi_term::Color;
 use rustyline::completion::{self, Completer, FilenameCompleter};
 use rustyline::error::ReadlineError;
@@ -113,11 +113,26 @@ fn paint_token_node(token_node: &TokenNode, line: &str) -> String {
         TokenNode::Delimited(..) => Color::White.paint(token_node.span().slice(line)),
         TokenNode::Operator(..) => Color::Purple.bold().paint(token_node.span().slice(line)),
         TokenNode::Pipeline(..) => Color::Blue.normal().paint(token_node.span().slice(line)),
-        TokenNode::Token(Spanned { item: RawToken::Integer(..), ..}) => Color::Purple.bold().paint(token_node.span().slice(line)),
-        TokenNode::Token(Spanned { item: RawToken::Size(..), ..}) => Color::Purple.bold().paint(token_node.span().slice(line)),
-        TokenNode::Token(Spanned { item: RawToken::String(..), ..}) => Color::Green.normal().paint(token_node.span().slice(line)),
-        TokenNode::Token(Spanned { item: RawToken::Variable(..), ..}) => Color::Yellow.bold().paint(token_node.span().slice(line)),
-        TokenNode::Token(Spanned { item: RawToken::Bare, ..}) => Color::Green.normal().paint(token_node.span().slice(line)),
+        TokenNode::Token(Spanned {
+            item: RawToken::Integer(..),
+            ..
+        }) => Color::Purple.bold().paint(token_node.span().slice(line)),
+        TokenNode::Token(Spanned {
+            item: RawToken::Size(..),
+            ..
+        }) => Color::Purple.bold().paint(token_node.span().slice(line)),
+        TokenNode::Token(Spanned {
+            item: RawToken::String(..),
+            ..
+        }) => Color::Green.normal().paint(token_node.span().slice(line)),
+        TokenNode::Token(Spanned {
+            item: RawToken::Variable(..),
+            ..
+        }) => Color::Yellow.bold().paint(token_node.span().slice(line)),
+        TokenNode::Token(Spanned {
+            item: RawToken::Bare,
+            ..
+        }) => Color::Green.normal().paint(token_node.span().slice(line)),
     };
 
     styled.to_string()