Getting ready for multiline scripts (#2737)

* WIP

* WIP

* WIP

* Tests are passing

* make parser more resilient

* lint
This commit is contained in:
Jonathan Turner 2020-11-10 05:27:07 +13:00 committed by GitHub
parent 0113661c81
commit 8df748463d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 570 additions and 471 deletions

View file

@ -393,52 +393,56 @@ pub async fn cli(mut context: EvaluationContext) -> Result<(), Box<dyn Error>> {
if let Some(prompt) = configuration.var("prompt") {
let prompt_line = prompt.as_string()?;
match nu_parser::lite_parse(&prompt_line, 0).map_err(ShellError::from) {
Ok(result) => {
let prompt_block = nu_parser::classify_block(&result, context.registry());
let (result, err) = nu_parser::lite_parse(&prompt_line, 0);
let env = context.get_env();
if err.is_some() {
use crate::git::current_branch;
format!(
"\x1b[32m{}{}\x1b[m> ",
cwd,
match current_branch() {
Some(s) => format!("({})", s),
None => "".to_string(),
}
)
} else {
let prompt_block = nu_parser::classify_block(&result, context.registry());
match run_block(
&prompt_block.block,
&mut context,
InputStream::empty(),
Scope::from_env(env),
)
.await
{
Ok(result) => match result.collect_string(Tag::unknown()).await {
Ok(string_result) => {
let errors = context.get_errors();
context.maybe_print_errors(Text::from(prompt_line));
context.clear_errors();
let env = context.get_env();
if !errors.is_empty() {
"> ".to_string()
} else {
string_result.item
}
}
Err(e) => {
crate::cli::print_err(e, &Text::from(prompt_line));
context.clear_errors();
match run_block(
&prompt_block.block,
&mut context,
InputStream::empty(),
Scope::from_env(env),
)
.await
{
Ok(result) => match result.collect_string(Tag::unknown()).await {
Ok(string_result) => {
let errors = context.get_errors();
context.maybe_print_errors(Text::from(prompt_line));
context.clear_errors();
if !errors.is_empty() {
"> ".to_string()
} else {
string_result.item
}
},
}
Err(e) => {
crate::cli::print_err(e, &Text::from(prompt_line));
context.clear_errors();
"> ".to_string()
}
}
}
Err(e) => {
crate::cli::print_err(e, &Text::from(prompt_line));
context.clear_errors();
},
Err(e) => {
crate::cli::print_err(e, &Text::from(prompt_line));
context.clear_errors();
"> ".to_string()
"> ".to_string()
}
}
}
} else {
@ -864,7 +868,10 @@ pub async fn parse_and_eval(line: &str, ctx: &mut EvaluationContext) -> Result<S
line
};
let lite_result = nu_parser::lite_parse(&line, 0)?;
let (lite_result, err) = nu_parser::lite_parse(&line, 0);
if let Some(err) = err {
return Err(err.into());
}
// TODO ensure the command whose examples we're testing is actually in the pipeline
let classified_block = nu_parser::classify_block(&lite_result, ctx.registry());
@ -897,13 +904,11 @@ pub async fn process_line(
let line = chomp_newline(line);
ctx.raw_input = line.to_string();
let result = match nu_parser::lite_parse(&line, 0) {
Err(err) => {
return LineResult::Error(line.to_string(), err.into());
}
let (result, err) = nu_parser::lite_parse(&line, 0);
Ok(val) => val,
};
if let Some(err) = err {
return LineResult::Error(line.to_string(), err.into());
}
debug!("=== Parsed ===");
debug!("{:#?}", result);
@ -1100,7 +1105,8 @@ pub fn print_err(err: ShellError, source: &Text) {
mod tests {
#[quickcheck]
fn quickcheck_parse(data: String) -> bool {
if let Ok(lite_block) = nu_parser::lite_parse(&data, 0) {
let (lite_block, err) = nu_parser::lite_parse(&data, 0);
if err.is_none() {
let context = crate::evaluation_context::EvaluationContext::basic().unwrap();
let _ = nu_parser::classify_block(&lite_block, context.registry());
}

View file

@ -288,10 +288,7 @@ mod tests {
registry: &dyn SignatureRegistry,
pos: usize,
) -> Vec<LocationType> {
let lite_block = match lite_parse(line, 0) {
Ok(v) => v,
Err(e) => e.partial.expect("lite_parse result"),
};
let (lite_block, _) = lite_parse(line, 0);
let block = classify_block(&lite_block, registry);

View file

@ -197,7 +197,10 @@ fn parse_line(line: &str, ctx: &mut EvaluationContext) -> Result<ClassifiedBlock
line
};
let lite_result = nu_parser::lite_parse(&line, 0)?;
let (lite_result, err) = nu_parser::lite_parse(&line, 0);
if let Some(err) = err {
return Err(err.into());
}
// TODO ensure the command whose examples we're testing is actually in the pipeline
let classified_block = nu_parser::classify_block(&lite_result, ctx.registry());

View file

@ -23,15 +23,10 @@ impl NuCompleter {
use completion::engine::LocationType;
let nu_context: &EvaluationContext = context.as_ref();
let lite_block = match nu_parser::lite_parse(line, 0) {
Ok(block) => Some(block),
Err(result) => result.partial,
};
let (lite_block, _) = nu_parser::lite_parse(line, 0);
let locations = lite_block
.map(|block| nu_parser::classify_block(&block, &nu_context.registry))
.map(|block| completion::engine::completion_location(line, &block.block, pos))
.unwrap_or_default();
let classified_block = nu_parser::classify_block(&lite_block, &nu_context.registry);
let locations = completion::engine::completion_location(line, &classified_block.block, pos);
let matcher = nu_data::config::config(Tag::unknown())
.ok()

View file

@ -121,10 +121,10 @@ impl rustyline::validate::Validator for NuValidator {
) -> rustyline::Result<rustyline::validate::ValidationResult> {
let src = ctx.input();
let lite_result = nu_parser::lite_parse(src, 0);
let (_, err) = nu_parser::lite_parse(src, 0);
if let Err(err) = lite_result {
if let nu_errors::ParseErrorReason::Eof { .. } = err.cause.reason() {
if let Some(err) = err {
if let nu_errors::ParseErrorReason::Eof { .. } = err.reason() {
return Ok(rustyline::validate::ValidationResult::Incomplete);
}
}

View file

@ -25,22 +25,21 @@ impl Painter {
registry: &dyn SignatureRegistry,
palette: &P,
) -> Cow<'l, str> {
let lite_block = nu_parser::lite_parse(line, 0);
let (lb, err) = nu_parser::lite_parse(line, 0);
match lite_block {
Err(_) => Cow::Borrowed(line),
Ok(lb) => {
let classified = nu_parser::classify_block(&lb, registry);
if err.is_some() {
Cow::Borrowed(line)
} else {
let classified = nu_parser::classify_block(&lb, registry);
let shapes = nu_parser::shapes(&classified.block);
let mut painter = Painter::new(line);
let shapes = nu_parser::shapes(&classified.block);
let mut painter = Painter::new(line);
for shape in shapes {
painter.paint_shape(&shape, palette);
}
Cow::Owned(painter.into_string())
for shape in shapes {
painter.paint_shape(&shape, palette);
}
Cow::Owned(painter.into_string())
}
}

View file

@ -1,19 +1,18 @@
use std::fmt::Debug;
// use std::fmt::Debug;
/// A combination of an informative parse error, and what has been successfully parsed so far
#[derive(Debug)]
pub struct ParseError<T: Debug> {
/// An informative cause for this parse error
pub cause: nu_errors::ParseError,
// A combination of an informative parse error, and what has been successfully parsed so far
// #[derive(Debug)]
// pub struct ParseError {
// /// An informative cause for this parse error
// pub cause: nu_errors::ParseError,
// // /// What has been successfully parsed, if anything
// // pub partial: Option<T>,
// }
/// What has been successfully parsed, if anything
pub partial: Option<T>,
}
// pub type ParseResult<T> = Result<T, ParseError<T>>;
pub type ParseResult<T> = Result<T, ParseError<T>>;
impl<T: Debug> From<ParseError<T>> for nu_errors::ShellError {
fn from(e: ParseError<T>) -> Self {
e.cause.into()
}
}
// impl<T: Debug> From<ParseError<T>> for nu_errors::ShellError {
// fn from(e: ParseError<T>) -> Self {
// e.cause.into()
// }
// }

View file

@ -5,7 +5,6 @@ mod path;
mod shapes;
mod signature;
pub use errors::{ParseError, ParseResult};
pub use lite_parse::{lite_parse, LiteBlock};
pub use parse::{classify_block, garbage, parse_full_column_path};
pub use path::expand_ndots;

View file

@ -3,27 +3,57 @@ use std::str::CharIndices;
use nu_source::{Span, Spanned, SpannedItem};
use crate::errors::{ParseError, ParseResult};
use nu_errors::ParseError;
type Input<'t> = Peekable<CharIndices<'t>>;
#[derive(Debug)]
pub struct Token {
pub contents: TokenContents,
pub span: Span,
}
impl Token {
pub fn new(contents: TokenContents, span: Span) -> Token {
Token { contents, span }
}
}
#[derive(Debug)]
pub enum TokenContents {
Bare(String),
Pipe,
Semicolon,
EOL,
}
#[derive(Debug, Clone)]
pub struct LiteCommand {
pub name: Spanned<String>,
pub args: Vec<Spanned<String>>,
pub parts: Vec<Spanned<String>>,
}
impl LiteCommand {
fn new(name: Spanned<String>) -> LiteCommand {
LiteCommand { name, args: vec![] }
fn new() -> LiteCommand {
LiteCommand { parts: vec![] }
}
pub fn is_empty(&self) -> bool {
self.parts.is_empty()
}
pub fn push(&mut self, item: Spanned<String>) {
self.parts.push(item)
}
pub(crate) fn span(&self) -> Span {
let start = self.name.span.start();
let end = if let Some(x) = self.args.last() {
let start = if let Some(x) = self.parts.first() {
x.span.start()
} else {
0
};
let end = if let Some(x) = self.parts.last() {
x.span.end()
} else {
self.name.span.end()
0
};
Span::new(start, end)
@ -35,10 +65,25 @@ pub struct LitePipeline {
pub commands: Vec<LiteCommand>,
}
impl Default for LitePipeline {
fn default() -> Self {
Self::new()
}
}
impl LitePipeline {
pub fn new() -> Self {
Self { commands: vec![] }
}
pub fn is_empty(&self) -> bool {
self.commands.is_empty()
}
pub fn push(&mut self, item: LiteCommand) {
self.commands.push(item)
}
pub(crate) fn span(&self) -> Span {
let start = if !self.commands.is_empty() {
self.commands[0].name.span.start()
self.commands[0].span().start()
} else {
0
};
@ -51,12 +96,63 @@ impl LitePipeline {
}
}
#[derive(Debug, Clone)]
pub struct LiteGroup {
pub pipelines: Vec<LitePipeline>,
}
impl Default for LiteGroup {
fn default() -> Self {
Self::new()
}
}
impl LiteGroup {
pub fn new() -> Self {
Self { pipelines: vec![] }
}
pub fn is_empty(&self) -> bool {
self.pipelines.is_empty()
}
pub fn push(&mut self, item: LitePipeline) {
self.pipelines.push(item)
}
pub(crate) fn span(&self) -> Span {
let start = if !self.pipelines.is_empty() {
self.pipelines[0].span().start()
} else {
0
};
if let Some((last, _)) = self.pipelines[..].split_last() {
Span::new(start, last.span().end())
} else {
Span::new(start, 0)
}
}
}
#[derive(Debug, Clone)]
pub struct LiteBlock {
pub block: Vec<LitePipeline>,
pub block: Vec<LiteGroup>,
}
impl Default for LiteBlock {
fn default() -> Self {
Self::new()
}
}
impl LiteBlock {
pub fn new() -> Self {
Self { block: vec![] }
}
pub fn is_empty(&self) -> bool {
self.block.is_empty()
}
pub fn push(&mut self, item: LiteGroup) {
self.block.push(item)
}
pub(crate) fn span(&self) -> Span {
let start = if !self.block.is_empty() {
self.block[0].span().start()
@ -72,22 +168,6 @@ impl LiteBlock {
}
}
impl From<Spanned<String>> for LiteCommand {
fn from(v: Spanned<String>) -> LiteCommand {
LiteCommand::new(v)
}
}
fn skip_whitespace(src: &mut Input) {
while let Some((_, x)) = src.peek() {
if x.is_whitespace() {
let _ = src.next();
} else {
break;
}
}
}
#[derive(Clone, Copy)]
enum BlockKind {
Paren,
@ -105,9 +185,11 @@ impl From<BlockKind> for char {
}
}
fn bare(src: &mut Input, span_offset: usize) -> ParseResult<Spanned<String>> {
skip_whitespace(src);
/// Finds the extents of a bare (un-classified) token, returning the string with its associated span,
/// along with any parse error that was discovered along the way.
/// Bare tokens are unparsed content separated by spaces or a command separator (like pipe or semicolon)
/// Bare tokens may be surrounded by quotes (single, double, or backtick) or braces (square, paren, curly)
pub fn bare(src: &mut Input, span_offset: usize) -> (Spanned<String>, Option<ParseError>) {
let mut bare = String::new();
let start_offset = if let Some((pos, _)) = src.peek() {
*pos
@ -158,16 +240,13 @@ fn bare(src: &mut Input, span_offset: usize) -> ParseResult<Spanned<String>> {
if let Some(block) = block_level.last() {
let delim: char = (*block).into();
let cause = nu_errors::ParseError::unexpected_eof(delim.to_string(), span);
let cause = ParseError::unexpected_eof(delim.to_string(), span);
while let Some(bk) = block_level.pop() {
bare.push(bk.into());
}
return Err(ParseError {
cause,
partial: Some(bare.spanned(span)),
});
return (bare.spanned(span), Some(cause));
}
if let Some(delimiter) = inside_quote {
@ -176,133 +255,161 @@ fn bare(src: &mut Input, span_offset: usize) -> ParseResult<Spanned<String>> {
// correct information from the non-lite parse.
bare.push(delimiter);
return Err(ParseError {
cause: nu_errors::ParseError::unexpected_eof(delimiter.to_string(), span),
partial: Some(bare.spanned(span)),
});
return (
bare.spanned(span),
Some(ParseError::unexpected_eof(delimiter.to_string(), span)),
);
}
if bare.is_empty() {
return Err(ParseError {
cause: nu_errors::ParseError::unexpected_eof("command", span),
partial: Some(bare.spanned(span)),
});
return (
bare.spanned(span),
Some(ParseError::unexpected_eof("command".to_string(), span)),
);
}
Ok(bare.spanned(span))
(bare.spanned(span), None)
}
fn command(src: &mut Input, span_offset: usize) -> ParseResult<LiteCommand> {
let mut cmd = match bare(src, span_offset) {
Ok(v) => LiteCommand::new(v),
Err(e) => {
return Err(ParseError {
cause: e.cause,
partial: e.partial.map(LiteCommand::new),
});
}
};
/// Breaks the input string into a vector of tokens. This tokenization only tries to classify separators like
/// semicolons, pipes, etc from external bare values (values that haven't been classified further)
/// Takes in a string and and offset, which is used to offset the spans created (for when this function is used to parse inner strings)
pub fn lex(input: &str, span_offset: usize) -> (Vec<Token>, Option<ParseError>) {
let mut char_indices = input.char_indices().peekable();
let mut error = None;
loop {
skip_whitespace(src);
let mut output = vec![];
if let Some((_, c)) = src.peek() {
// The first character tells us a lot about each argument
match c {
';' => {
// this is the end of the command and the end of the pipeline
break;
}
'|' => {
let _ = src.next();
if let Some((pos, next_c)) = src.peek() {
if *next_c == '|' {
// this isn't actually a pipeline but a comparison
let span = Span::new(pos - 1 + span_offset, pos + 1 + span_offset);
cmd.args.push("||".to_string().spanned(span));
let _ = src.next();
} else {
// this is the end of this command
break;
}
} else {
// this is the end of this command
break;
}
}
_ => {
// basic argument
match bare(src, span_offset) {
Ok(v) => {
cmd.args.push(v);
}
Err(e) => {
if let Some(v) = e.partial {
cmd.args.push(v);
}
return Err(ParseError {
cause: e.cause,
partial: Some(cmd),
});
}
}
while let Some((idx, c)) = char_indices.peek() {
if *c == '|' {
let idx = *idx;
let prev_idx = idx;
let _ = char_indices.next();
if let Some((idx, c)) = char_indices.peek() {
if *c == '|' {
// we have '||' instead of '|'
let idx = *idx;
let _ = char_indices.next();
output.push(Token::new(
TokenContents::Bare("||".into()),
Span::new(span_offset + prev_idx, span_offset + idx + 1),
));
continue;
}
}
output.push(Token::new(
TokenContents::Pipe,
Span::new(span_offset + idx, span_offset + idx + 1),
));
} else if *c == ';' {
let idx = *idx;
let _ = char_indices.next();
output.push(Token::new(
TokenContents::Semicolon,
Span::new(span_offset + idx, span_offset + idx + 1),
));
} else if *c == '\n' || *c == '\r' {
let idx = *idx;
let _ = char_indices.next();
output.push(Token::new(
TokenContents::EOL,
Span::new(span_offset + idx, span_offset + idx + 1),
));
} else if c.is_whitespace() {
let _ = char_indices.next();
} else {
break;
let (result, err) = bare(&mut char_indices, span_offset);
if error.is_none() {
error = err;
}
let Spanned { item, span } = result;
output.push(Token::new(TokenContents::Bare(item), span));
}
}
Ok(cmd)
(output, error)
}
fn pipeline(src: &mut Input, span_offset: usize) -> ParseResult<LiteBlock> {
let mut block = vec![];
let mut commands = vec![];
fn group(tokens: Vec<Token>) -> (LiteBlock, Option<ParseError>) {
let mut groups = vec![];
let mut group = LiteGroup::new();
let mut pipeline = LitePipeline::new();
let mut command = LiteCommand::new();
skip_whitespace(src);
while src.peek().is_some() {
// If there is content there, let's parse it
let cmd = match command(src, span_offset) {
Ok(v) => v,
Err(e) => {
if let Some(partial) = e.partial {
commands.push(partial);
block.push(LitePipeline { commands });
for token in tokens {
match token.contents {
TokenContents::EOL => {
if !command.is_empty() {
pipeline.push(command);
command = LiteCommand::new();
}
if !pipeline.is_empty() {
group.push(pipeline);
pipeline = LitePipeline::new();
}
if !group.is_empty() {
groups.push(group);
group = LiteGroup::new();
}
return Err(ParseError {
cause: e.cause,
partial: Some(LiteBlock { block }),
});
}
};
TokenContents::Pipe => {
if !command.is_empty() {
pipeline.push(command);
command = LiteCommand::new();
} else {
let mut block = LiteBlock::new();
block.block = groups;
commands.push(cmd);
skip_whitespace(src);
if let Some((_, ';')) = src.peek() {
let _ = src.next();
if !commands.is_empty() {
block.push(LitePipeline { commands });
commands = vec![];
return (
block,
Some(ParseError::extra_tokens(
"|".to_string().spanned(token.span),
)),
);
}
}
TokenContents::Semicolon => {
if !command.is_empty() {
pipeline.push(command);
command = LiteCommand::new();
}
if !pipeline.is_empty() {
group.push(pipeline);
pipeline = LitePipeline::new();
}
}
TokenContents::Bare(bare) => {
command.push(bare.spanned(token.span));
}
}
}
if !commands.is_empty() {
block.push(LitePipeline { commands });
if !command.is_empty() {
pipeline.push(command);
}
if !pipeline.is_empty() {
group.push(pipeline);
}
if !group.is_empty() {
groups.push(group);
}
Ok(LiteBlock { block })
let mut block = LiteBlock::new();
block.block = groups;
(block, None)
}
pub fn lite_parse(src: &str, span_offset: usize) -> ParseResult<LiteBlock> {
pipeline(&mut src.char_indices().peekable(), span_offset)
pub fn lite_parse(src: &str, span_offset: usize) -> (LiteBlock, Option<ParseError>) {
let mut error = None;
let (output, err) = lex(src, span_offset);
if err.is_some() {
error = err;
}
let (group_output, err) = group(output);
if error.is_none() {
error = err;
}
(group_output, error)
}
#[cfg(test)]
@ -320,140 +427,136 @@ mod tests {
fn simple_1() {
let input = "foo bar baz";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0).unwrap();
let (result, err) = lex(input, 0);
assert_eq!(result.span, span(0, 3));
assert!(err.is_none());
assert_eq!(result[0].span, span(0, 3));
}
#[test]
fn simple_2() {
let input = "'foo bar' baz";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0).unwrap();
let (result, err) = lex(input, 0);
assert_eq!(result.span, span(0, 9));
assert!(err.is_none());
assert_eq!(result[0].span, span(0, 9));
}
#[test]
fn simple_3() {
let input = "'foo\" bar' baz";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0).unwrap();
let (result, err) = lex(input, 0);
assert_eq!(result.span, span(0, 10));
assert!(err.is_none());
assert_eq!(result[0].span, span(0, 10));
}
#[test]
fn simple_4() {
let input = "[foo bar] baz";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0).unwrap();
let (result, err) = lex(input, 0);
assert_eq!(result.span, span(0, 9));
assert!(err.is_none());
assert_eq!(result[0].span, span(0, 9));
}
#[test]
fn simple_5() {
let input = "'foo 'bar baz";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0).unwrap();
let (result, err) = lex(input, 0);
assert_eq!(result.span, span(0, 9));
assert!(err.is_none());
assert_eq!(result[0].span, span(0, 9));
}
#[test]
fn simple_6() {
let input = "''foo baz";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0).unwrap();
let (result, err) = lex(input, 0);
assert_eq!(result.span, span(0, 5));
assert!(err.is_none());
assert_eq!(result[0].span, span(0, 5));
}
#[test]
fn simple_7() {
let input = "'' foo";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0).unwrap();
let (result, err) = lex(input, 0);
assert_eq!(result.span, span(0, 2));
assert!(err.is_none());
assert_eq!(result[0].span, span(0, 2));
}
#[test]
fn simple_8() {
let input = " '' foo";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0).unwrap();
let (result, err) = lex(input, 0);
assert_eq!(result.span, span(1, 3));
assert!(err.is_none());
assert_eq!(result[0].span, span(1, 3));
}
#[test]
fn simple_9() {
let input = " 'foo' foo";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0).unwrap();
let (result, err) = lex(input, 0);
assert_eq!(result.span, span(1, 6));
assert!(err.is_none());
assert_eq!(result[0].span, span(1, 6));
}
#[test]
fn simple_10() {
let input = "[foo, bar]";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0).unwrap();
let (result, err) = lex(input, 0);
assert_eq!(result.span, span(0, 10));
assert!(err.is_none());
assert_eq!(result[0].span, span(0, 10));
}
#[test]
fn ignore_future() {
let input = "foo 'bar";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0).unwrap();
let (result, _) = lex(input, 0);
assert_eq!(result.span, span(0, 3));
assert_eq!(result[0].span, span(0, 3));
}
#[test]
fn invalid_1() {
let input = "'foo bar";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0);
let (_, err) = lex(input, 0);
assert_eq!(result.is_ok(), false);
assert!(err.is_some());
}
#[test]
fn invalid_2() {
let input = "'bar";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0);
let (_, err) = lex(input, 0);
assert_eq!(result.is_ok(), false);
assert!(err.is_some());
}
#[test]
fn invalid_4() {
let input = " 'bar";
let input = &mut input.char_indices().peekable();
let result = bare(input, 0);
let (_, err) = lex(input, 0);
assert_eq!(result.is_ok(), false);
assert!(err.is_some());
}
}
@ -462,39 +565,58 @@ mod tests {
#[test]
fn pipeline() {
let result = lite_parse("cmd1 | cmd2 ; deploy", 0).unwrap();
let (result, err) = lite_parse("cmd1 | cmd2 ; deploy", 0);
assert!(err.is_none());
assert_eq!(result.span(), span(0, 20));
assert_eq!(result.block[0].span(), span(0, 11));
assert_eq!(result.block[1].span(), span(14, 20));
assert_eq!(result.block[0].pipelines[0].span(), span(0, 11));
assert_eq!(result.block[0].pipelines[1].span(), span(14, 20));
}
#[test]
fn simple_1() {
let result = lite_parse("foo", 0).unwrap();
let (result, err) = lite_parse("foo", 0);
assert!(err.is_none());
assert_eq!(result.block.len(), 1);
assert_eq!(result.block[0].commands.len(), 1);
assert_eq!(result.block[0].commands[0].name.span, span(0, 3));
assert_eq!(result.block[0].pipelines.len(), 1);
assert_eq!(result.block[0].pipelines[0].commands.len(), 1);
assert_eq!(result.block[0].pipelines[0].commands[0].parts.len(), 1);
assert_eq!(
result.block[0].pipelines[0].commands[0].parts[0].span,
span(0, 3)
);
}
#[test]
fn simple_offset() {
let result = lite_parse("foo", 10).unwrap();
assert_eq!(result.block.len(), 1);
assert_eq!(result.block[0].commands.len(), 1);
assert_eq!(result.block[0].commands[0].name.span, span(10, 13));
let (result, err) = lite_parse("foo", 10);
assert!(err.is_none());
assert_eq!(result.block[0].pipelines.len(), 1);
assert_eq!(result.block[0].pipelines[0].commands.len(), 1);
assert_eq!(result.block[0].pipelines[0].commands[0].parts.len(), 1);
assert_eq!(
result.block[0].pipelines[0].commands[0].parts[0].span,
span(10, 13)
);
}
#[test]
fn incomplete_result() {
let result = lite_parse("my_command \"foo' --test", 10).unwrap_err();
assert!(matches!(result.cause.reason(), nu_errors::ParseErrorReason::Eof { .. }));
let (result, err) = lite_parse("my_command \"foo' --test", 10);
assert!(matches!(err.unwrap().reason(), nu_errors::ParseErrorReason::Eof { .. }));
let result = result.partial.unwrap();
assert_eq!(result.block.len(), 1);
assert_eq!(result.block[0].commands.len(), 1);
assert_eq!(result.block[0].commands[0].name.item, "my_command");
assert_eq!(result.block[0].commands[0].args.len(), 1);
assert_eq!(result.block[0].commands[0].args[0].item, "\"foo' --test\"");
assert_eq!(result.block[0].pipelines.len(), 1);
assert_eq!(result.block[0].pipelines[0].commands.len(), 1);
assert_eq!(result.block[0].pipelines[0].commands[0].parts.len(), 2);
assert_eq!(
result.block[0].pipelines[0].commands[0].parts[0].item,
"my_command"
);
assert_eq!(
result.block[0].pipelines[0].commands[0].parts[1].item,
"\"foo' --test\""
);
}
}
}

View file

@ -388,9 +388,9 @@ fn parse_invocation(
.collect();
// We haven't done much with the inner string, so let's go ahead and work with it
let lite_block = match lite_parse(&string, lite_arg.span.start() + 2) {
Ok(lp) => lp,
Err(e) => return (garbage(lite_arg.span), Some(e.cause)),
let (lite_block, err) = lite_parse(&string, lite_arg.span.start() + 2);
if err.is_some() {
return (garbage(lite_arg.span), err);
};
let classified_block = classify_block(&lite_block, registry);
@ -641,38 +641,22 @@ fn parse_list(
}
let lite_pipeline = &lite_block.block[0];
let mut output = vec![];
for lite_inner in &lite_pipeline.commands {
let item = if lite_inner.name.ends_with(',') {
let mut str: String = lite_inner.name.item.clone();
str.pop();
str.spanned(Span::new(
lite_inner.name.span.start(),
lite_inner.name.span.end() - 1,
))
} else {
lite_inner.name.clone()
};
for lite_pipeline in &lite_pipeline.pipelines {
for lite_inner in &lite_pipeline.commands {
for part in &lite_inner.parts {
let item = if part.ends_with(',') {
let mut str: String = part.item.clone();
str.pop();
str.spanned(Span::new(part.span.start(), part.span.end() - 1))
} else {
part.clone()
};
let (part, err) = parse_arg(SyntaxShape::Any, registry, &item);
output.push(part);
let (arg, err) = parse_arg(SyntaxShape::Any, registry, &item);
output.push(arg);
if error.is_none() {
error = err;
}
for arg in &lite_inner.args {
let item = if arg.ends_with(',') {
let mut str: String = arg.item.clone();
str.pop();
str.spanned(Span::new(arg.span.start(), arg.span.end() - 1))
} else {
arg.clone()
};
let (arg, err) = parse_arg(SyntaxShape::Any, registry, &item);
output.push(arg);
if error.is_none() {
error = err;
if error.is_none() {
error = err;
}
}
}
}
@ -711,18 +695,19 @@ fn parse_table(
let mut output = vec![];
// Header
let lite_pipeline = &lite_block.block[0];
let lite_group = &lite_block.block[0];
let lite_pipeline = &lite_group.pipelines[0];
let lite_inner = &lite_pipeline.commands[0];
let (string, err) = verify_and_strip(&lite_inner.name, '[', ']');
let (string, err) = verify_and_strip(&lite_inner.parts[0], '[', ']');
if error.is_none() {
error = err;
}
let lite_header = match lite_parse(&string, lite_inner.name.span.start() + 1) {
Ok(lb) => lb,
Err(e) => return (garbage(lite_inner.name.span), Some(e.cause)),
};
let (lite_header, err) = lite_parse(&string, lite_inner.parts[0].span.start() + 1);
if err.is_some() {
return (garbage(lite_inner.span()), err);
}
let (headers, err) = parse_list(&lite_header, registry);
if error.is_none() {
@ -730,34 +715,18 @@ fn parse_table(
}
// Cells
let lite_rows = &lite_block.block[1];
let lite_rows = &lite_group.pipelines[1];
let lite_cells = &lite_rows.commands[0];
let (string, err) = verify_and_strip(&lite_cells.name, '[', ']');
if error.is_none() {
error = err;
}
let lite_cell = match lite_parse(&string, lite_cells.name.span.start() + 1) {
Ok(lb) => lb,
Err(e) => return (garbage(lite_cells.name.span), Some(e.cause)),
};
let (inner_cell, err) = parse_list(&lite_cell, registry);
if error.is_none() {
error = err;
}
output.push(inner_cell);
for arg in &lite_cells.args {
for arg in &lite_cells.parts {
let (string, err) = verify_and_strip(&arg, '[', ']');
if error.is_none() {
error = err;
}
let lite_cell = match lite_parse(&string, arg.span.start() + 1) {
Ok(lb) => lb,
Err(e) => return (garbage(arg.span), Some(e.cause)),
};
let (lite_cell, err) = lite_parse(&string, arg.span.start() + 1);
if err.is_some() {
return (garbage(arg.span), err);
}
let (inner_cell, err) = parse_list(&lite_cell, registry);
if error.is_none() {
error = err;
@ -880,24 +849,25 @@ fn parse_arg(
let string: String = chars.collect();
// We haven't done much with the inner string, so let's go ahead and work with it
let lite_block = match lite_parse(&string, lite_arg.span.start() + 1) {
Ok(lb) => lb,
Err(e) => return (garbage(lite_arg.span), Some(e.cause)),
};
let (lite_block, err) = lite_parse(&string, lite_arg.span.start() + 1);
if err.is_some() {
return (garbage(lite_arg.span), err);
}
let lite_groups = &lite_block.block;
if lite_block.block.is_empty() {
if lite_groups.is_empty() {
return (
SpannedExpression::new(Expression::List(vec![]), lite_arg.span),
None,
);
}
if lite_block.block.len() == 1 {
if lite_groups[0].pipelines.len() == 1 {
let (items, err) = parse_list(&lite_block, registry);
(
SpannedExpression::new(Expression::List(items), lite_arg.span),
err,
)
} else if lite_block.block.len() == 2 {
} else if lite_groups[0].pipelines.len() == 2 {
parse_table(&lite_block, registry, lite_arg.span)
} else {
(
@ -926,10 +896,10 @@ fn parse_arg(
let string: String = chars.collect();
// We haven't done much with the inner string, so let's go ahead and work with it
let lite_block = match lite_parse(&string, lite_arg.span.start() + 1) {
Ok(lp) => lp,
Err(e) => return (garbage(lite_arg.span), Some(e.cause)),
};
let (lite_block, err) = lite_parse(&string, lite_arg.span.start() + 1);
if err.is_some() {
return (garbage(lite_arg.span), err);
}
let classified_block = classify_block(&lite_block, registry);
let error = classified_block.failed;
@ -1147,10 +1117,10 @@ fn parse_parenthesized_expression(
let string: String = chars.collect();
// We haven't done much with the inner string, so let's go ahead and work with it
let lite_block = match lite_parse(&string, lite_arg.span.start() + 1) {
Ok(lb) => lb,
Err(e) => return (garbage(lite_arg.span), Some(e.cause)),
};
let (lite_block, err) = lite_parse(&string, lite_arg.span.start() + 1);
if err.is_some() {
return (garbage(lite_arg.span), err);
}
if lite_block.block.len() != 1 {
return (
@ -1162,9 +1132,10 @@ fn parse_parenthesized_expression(
let mut lite_pipeline = lite_block.block[0].clone();
let mut collection = vec![];
for lite_cmd in lite_pipeline.commands.iter_mut() {
collection.push(lite_cmd.name.clone());
collection.append(&mut lite_cmd.args);
for lite_pipeline in lite_pipeline.pipelines.iter_mut() {
for lite_cmd in lite_pipeline.commands.iter_mut() {
collection.append(&mut lite_cmd.parts);
}
}
let (_, expr, err) =
parse_math_expression(0, &collection[..], registry, shorthand_mode);
@ -1351,23 +1322,23 @@ fn parse_positional_argument(
// A condition can take up multiple arguments, as we build the operation as <arg> <operator> <arg>
// We need to do this here because in parse_arg, we have access to only one arg at a time
if idx < lite_cmd.args.len() {
if lite_cmd.args[idx].item.starts_with('{') {
if idx < lite_cmd.parts.len() {
if lite_cmd.parts[idx].item.starts_with('{') {
// It's an explicit math expression, so parse it deeper in
let (arg, err) = parse_arg(SyntaxShape::Math, registry, &lite_cmd.args[idx]);
let (arg, err) = parse_arg(SyntaxShape::Math, registry, &lite_cmd.parts[idx]);
if error.is_none() {
error = err;
}
arg
} else {
let end_idx = if lite_cmd.args.len() > remaining_positionals {
lite_cmd.args.len() - remaining_positionals
let end_idx = if (lite_cmd.parts.len() - 1) > remaining_positionals {
lite_cmd.parts.len() - remaining_positionals
} else {
lite_cmd.args.len()
lite_cmd.parts.len()
};
let (new_idx, arg, err) =
parse_math_expression(idx, &lite_cmd.args[idx..end_idx], registry, true);
parse_math_expression(idx, &lite_cmd.parts[idx..end_idx], registry, true);
let span = arg.span;
let mut commands = hir::Commands::new(span);
@ -1386,7 +1357,7 @@ fn parse_positional_argument(
} else {
if error.is_none() {
error = Some(ParseError::argument_error(
lite_cmd.name.clone(),
lite_cmd.parts[0].clone(),
ArgumentError::MissingMandatoryPositional("condition".into()),
))
}
@ -1394,7 +1365,7 @@ fn parse_positional_argument(
}
}
PositionalType::Mandatory(_, shape) | PositionalType::Optional(_, shape) => {
let (arg, err) = parse_arg(*shape, registry, &lite_cmd.args[idx]);
let (arg, err) = parse_arg(*shape, registry, &lite_cmd.parts[idx]);
if error.is_none() {
error = err;
}
@ -1416,14 +1387,17 @@ fn parse_internal_command(
) -> (InternalCommand, Option<ParseError>) {
// This is a known internal command, so we need to work with the arguments and parse them according to the expected types
let (name, name_span) = if idx == 0 {
(lite_cmd.name.item.clone(), lite_cmd.name.span)
} else {
(
format!("{} {}", lite_cmd.name.item, lite_cmd.args[0].item),
Span::new(lite_cmd.name.span.start(), lite_cmd.args[0].span.end()),
)
};
let (name, name_span) = (
lite_cmd.parts[0..(idx + 1)]
.iter()
.map(|x| x.item.clone())
.collect::<Vec<String>>()
.join(" "),
Span::new(
lite_cmd.parts[0].span.start(),
lite_cmd.parts[idx].span.end(),
),
);
let mut internal_command = InternalCommand::new(name, name_span, lite_cmd.span());
internal_command.args.set_initial_flags(&signature);
@ -1432,32 +1406,33 @@ fn parse_internal_command(
let mut named = NamedArguments::new();
let mut positional = vec![];
let mut error = None;
idx += 1; // Start where the arguments begin
while idx < lite_cmd.args.len() {
if lite_cmd.args[idx].item.starts_with('-') && lite_cmd.args[idx].item.len() > 1 {
while idx < lite_cmd.parts.len() {
if lite_cmd.parts[idx].item.starts_with('-') && lite_cmd.parts[idx].item.len() > 1 {
let (named_types, err) =
get_flags_from_flag(&signature, &lite_cmd.name, &lite_cmd.args[idx]);
get_flags_from_flag(&signature, &lite_cmd.parts[0], &lite_cmd.parts[idx]);
if err.is_none() {
for (full_name, named_type) in &named_types {
match named_type {
NamedType::Mandatory(_, shape) | NamedType::Optional(_, shape) => {
if idx == lite_cmd.args.len() {
if idx == lite_cmd.parts.len() {
// Oops, we're missing the argument to our named argument
if error.is_none() {
error = Some(ParseError::argument_error(
lite_cmd.name.clone(),
lite_cmd.parts[0].clone(),
ArgumentError::MissingValueForName(format!("{:?}", shape)),
));
}
} else {
idx += 1;
if lite_cmd.args.len() > idx {
if lite_cmd.parts.len() > idx {
let (arg, err) =
parse_arg(*shape, registry, &lite_cmd.args[idx]);
parse_arg(*shape, registry, &lite_cmd.parts[idx]);
named.insert_mandatory(
full_name.clone(),
lite_cmd.args[idx - 1].span,
lite_cmd.parts[idx - 1].span,
arg,
);
@ -1466,7 +1441,7 @@ fn parse_internal_command(
}
} else if error.is_none() {
error = Some(ParseError::argument_error(
lite_cmd.name.clone(),
lite_cmd.parts[0].clone(),
ArgumentError::MissingValueForName(full_name.to_owned()),
));
}
@ -1475,13 +1450,13 @@ fn parse_internal_command(
NamedType::Switch(_) => {
named.insert_switch(
full_name.clone(),
Some(Flag::new(FlagKind::Longhand, lite_cmd.args[idx].span)),
Some(Flag::new(FlagKind::Longhand, lite_cmd.parts[idx].span)),
);
}
}
}
} else {
positional.push(garbage(lite_cmd.args[idx].span));
positional.push(garbage(lite_cmd.parts[idx].span));
if error.is_none() {
error = err;
@ -1506,7 +1481,7 @@ fn parse_internal_command(
positional.push(arg);
current_positional += 1;
} else if let Some((rest_type, _)) = &signature.rest_positional {
let (arg, err) = parse_arg(*rest_type, registry, &lite_cmd.args[idx]);
let (arg, err) = parse_arg(*rest_type, registry, &lite_cmd.parts[idx]);
if error.is_none() {
error = err;
}
@ -1514,12 +1489,12 @@ fn parse_internal_command(
positional.push(arg);
current_positional += 1;
} else {
positional.push(garbage(lite_cmd.args[idx].span));
positional.push(garbage(lite_cmd.parts[idx].span));
if error.is_none() {
error = Some(ParseError::argument_error(
lite_cmd.name.clone(),
ArgumentError::UnexpectedArgument(lite_cmd.args[idx].clone()),
lite_cmd.parts[0].clone(),
ArgumentError::UnexpectedArgument(lite_cmd.parts[idx].clone()),
));
}
}
@ -1539,7 +1514,7 @@ fn parse_internal_command(
if !named.named.contains_key("help") {
let (_, name) = &signature.positional[positional.len()];
error = Some(ParseError::argument_error(
lite_cmd.name.clone(),
lite_cmd.parts[0].clone(),
ArgumentError::MissingMandatoryPositional(name.to_owned()),
));
}
@ -1568,9 +1543,11 @@ fn classify_pipeline(
let mut iter = lite_pipeline.commands.iter().peekable();
while let Some(lite_cmd) = iter.next() {
if lite_cmd.name.item.starts_with('^') {
let name = lite_cmd
.name
if lite_cmd.parts.is_empty() {
continue;
}
if lite_cmd.parts[0].item.starts_with('^') {
let name = lite_cmd.parts[0]
.clone()
.map(|v| v.chars().skip(1).collect::<String>());
// TODO this is the same as the `else` branch below, only the name differs. Find a way
@ -1584,7 +1561,7 @@ fn classify_pipeline(
}
args.push(name);
for lite_arg in &lite_cmd.args {
for lite_arg in &lite_cmd.parts[1..] {
let (expr, err) = parse_external_arg(registry, lite_arg);
if error.is_none() {
error = err;
@ -1610,15 +1587,16 @@ fn classify_pipeline(
},
},
}))
} else if lite_cmd.name.item == "=" {
let expr = if !lite_cmd.args.is_empty() {
let (_, expr, err) = parse_math_expression(0, &lite_cmd.args[0..], registry, false);
} else if lite_cmd.parts[0].item == "=" {
let expr = if lite_cmd.parts.len() > 1 {
let (_, expr, err) =
parse_math_expression(0, &lite_cmd.parts[1..], registry, false);
error = error.or(err);
expr
} else {
error = error.or_else(|| {
Some(ParseError::argument_error(
lite_cmd.name.clone(),
lite_cmd.parts[0].clone(),
ArgumentError::MissingMandatoryPositional("an expression".into()),
))
});
@ -1626,11 +1604,12 @@ fn classify_pipeline(
};
commands.push(ClassifiedCommand::Expr(Box::new(expr)))
} else {
if !lite_cmd.args.is_empty() {
if lite_cmd.parts.len() > 1 {
// Check if it's a sub-command
if let Some(signature) =
registry.get(&format!("{} {}", lite_cmd.name.item, lite_cmd.args[0].item))
{
if let Some(signature) = registry.get(&format!(
"{} {}",
lite_cmd.parts[0].item, lite_cmd.parts[1].item
)) {
let (mut internal_command, err) =
parse_internal_command(&lite_cmd, registry, &signature, 1);
@ -1646,7 +1625,7 @@ fn classify_pipeline(
}
// Check if it's an internal command
if let Some(signature) = registry.get(&lite_cmd.name.item) {
if let Some(signature) = registry.get(&lite_cmd.parts[0].item) {
let (mut internal_command, err) =
parse_internal_command(&lite_cmd, registry, &signature, 0);
@ -1660,7 +1639,7 @@ fn classify_pipeline(
continue;
}
let name = lite_cmd.name.clone().map(|v| {
let name = lite_cmd.parts[0].clone().map(|v| {
let trimmed = trim_quotes(&v);
expand_path(&trimmed).to_string()
});
@ -1674,7 +1653,7 @@ fn classify_pipeline(
}
args.push(name);
for lite_arg in &lite_cmd.args {
for lite_arg in &lite_cmd.parts[1..] {
let (expr, err) = parse_external_arg(registry, lite_arg);
if error.is_none() {
error = err;
@ -1712,33 +1691,31 @@ fn expand_shorthand_forms(
lite_pipeline: &LitePipeline,
) -> (LitePipeline, Option<SpannedKeyValue>, Option<ParseError>) {
if !lite_pipeline.commands.is_empty() {
if lite_pipeline.commands[0].name.item == "=" {
if lite_pipeline.commands[0].parts[0].item == "=" {
(lite_pipeline.clone(), None, None)
} else if lite_pipeline.commands[0].name.contains('=') {
let assignment: Vec<_> = lite_pipeline.commands[0].name.split('=').collect();
} else if lite_pipeline.commands[0].parts[0].contains('=') {
let assignment: Vec<_> = lite_pipeline.commands[0].parts[0].split('=').collect();
if assignment.len() != 2 {
(
lite_pipeline.clone(),
None,
Some(ParseError::mismatch(
"environment variable assignment",
lite_pipeline.commands[0].name.clone(),
lite_pipeline.commands[0].parts[0].clone(),
)),
)
} else {
let original_span = lite_pipeline.commands[0].name.span;
let original_span = lite_pipeline.commands[0].parts[0].span;
let env_value = trim_quotes(assignment[1]);
let (variable_name, value) = (assignment[0], env_value);
let mut lite_pipeline = lite_pipeline.clone();
if !lite_pipeline.commands[0].args.is_empty() {
let new_lite_command_name = lite_pipeline.commands[0].args[0].clone();
let mut new_lite_command_args = lite_pipeline.commands[0].args.clone();
new_lite_command_args.remove(0);
if !lite_pipeline.commands[0].parts.len() > 1 {
let mut new_lite_command_parts = lite_pipeline.commands[0].parts.clone();
new_lite_command_parts.remove(0);
lite_pipeline.commands[0].name = new_lite_command_name;
lite_pipeline.commands[0].args = new_lite_command_args;
lite_pipeline.commands[0].parts = new_lite_command_parts;
(
lite_pipeline,
@ -1754,7 +1731,7 @@ fn expand_shorthand_forms(
None,
Some(ParseError::mismatch(
"a command following variable",
lite_pipeline.commands[0].name.clone(),
lite_pipeline.commands[0].parts[0].clone(),
)),
)
}
@ -1771,61 +1748,63 @@ pub fn classify_block(lite_block: &LiteBlock, registry: &dyn SignatureRegistry)
let mut command_list = vec![];
let mut error = None;
for lite_pipeline in &lite_block.block {
let (lite_pipeline, vars, err) = expand_shorthand_forms(lite_pipeline);
if error.is_none() {
error = err;
}
let (pipeline, err) = classify_pipeline(&lite_pipeline, registry);
let pipeline = if let Some(vars) = vars {
let span = pipeline.commands.span;
let block = hir::Block::new(vec![], vec![pipeline.commands.clone()], span);
let mut call = hir::Call::new(
Box::new(SpannedExpression {
expr: Expression::string("with-env".to_string()),
span,
}),
span,
);
call.positional = Some(vec![
SpannedExpression {
expr: Expression::List(vec![
SpannedExpression {
expr: Expression::string(vars.0.item),
span: vars.0.span,
},
SpannedExpression {
expr: Expression::string(vars.1.item),
span: vars.1.span,
},
]),
span: Span::new(vars.0.span.start(), vars.1.span.end()),
},
SpannedExpression {
expr: Expression::Block(block),
span,
},
]);
let classified_with_env = ClassifiedCommand::Internal(InternalCommand {
name: "with-env".to_string(),
name_span: Span::unknown(),
args: call,
});
ClassifiedPipeline {
commands: Commands {
list: vec![classified_with_env],
span,
},
for lite_group in &lite_block.block {
for lite_pipeline in &lite_group.pipelines {
let (lite_pipeline, vars, err) = expand_shorthand_forms(lite_pipeline);
if error.is_none() {
error = err;
}
} else {
pipeline
};
command_list.push(pipeline.commands);
if error.is_none() {
error = err;
let (pipeline, err) = classify_pipeline(&lite_pipeline, registry);
let pipeline = if let Some(vars) = vars {
let span = pipeline.commands.span;
let block = hir::Block::new(vec![], vec![pipeline.commands.clone()], span);
let mut call = hir::Call::new(
Box::new(SpannedExpression {
expr: Expression::string("with-env".to_string()),
span,
}),
span,
);
call.positional = Some(vec![
SpannedExpression {
expr: Expression::List(vec![
SpannedExpression {
expr: Expression::string(vars.0.item),
span: vars.0.span,
},
SpannedExpression {
expr: Expression::string(vars.1.item),
span: vars.1.span,
},
]),
span: Span::new(vars.0.span.start(), vars.1.span.end()),
},
SpannedExpression {
expr: Expression::Block(block),
span,
},
]);
let classified_with_env = ClassifiedCommand::Internal(InternalCommand {
name: "with-env".to_string(),
name_span: Span::unknown(),
args: call,
});
ClassifiedPipeline {
commands: Commands {
list: vec![classified_with_env],
span,
},
}
} else {
pipeline
};
command_list.push(pipeline.commands);
if error.is_none() {
error = err;
}
}
}
let block = Block::new(vec![], command_list, lite_block.span());