From 6e222eec2b0a0af4a0872f243d718dfe56206c67 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Fri, 14 Jun 2019 19:59:35 -0700 Subject: [PATCH] More delimiters --- src/parser/parse2/parser.rs | 226 +++++++++++++++++++----- src/parser/parse2/token_tree.rs | 31 +++- src/parser/parse2/token_tree_builder.rs | 188 +++++++++++++++++++- 3 files changed, 392 insertions(+), 53 deletions(-) diff --git a/src/parser/parse2/parser.rs b/src/parser/parse2/parser.rs index 267cee78ed..5a2f6cc130 100644 --- a/src/parser/parse2/parser.rs +++ b/src/parser/parse2/parser.rs @@ -1,6 +1,8 @@ #![allow(unused)] -use crate::parser::parse2::{flag::*, operator::*, span::*, token_tree::*, tokens::*, unit::*}; +use crate::parser::parse2::{ + flag::*, operator::*, span::*, token_tree::*, token_tree_builder::*, tokens::*, unit::*, +}; use nom; use nom::dbg; use nom::types::CompleteStr; @@ -12,12 +14,13 @@ type NomSpan<'a> = LocatedSpan>; macro_rules! operator { ($name:tt : $token:tt ) => { - named!($name( NomSpan ) -> Token, + named!($name( NomSpan ) -> TokenNode, do_parse!( l: position!() >> t: tag!(stringify!($token)) >> r: position!() - >> (Spanned::from_nom(RawToken::Operator(Operator::from_str(t.fragment.0).unwrap()), l, r)) + >> (TokenTreeBuilder::spanned_op(t.fragment.0, (l, r))) + // >> (Spanned::from_nom(RawToken::Operator(Operator::from_str(t.fragment.0).unwrap()), l, r)) ) ); }; @@ -40,20 +43,20 @@ named!(pub raw_integer( NomSpan ) -> Spanned, ) ); -named!(pub integer( NomSpan ) -> Token, +named!(pub integer( NomSpan ) -> TokenNode, do_parse!( int: raw_integer - >> (int.map(|i| RawToken::Integer(i))) + >> (TokenTreeBuilder::spanned_int(*int, int.span)) ) ); -named!(pub operator( NomSpan ) -> Token, +named!(pub operator( NomSpan ) -> TokenNode, alt!( gte | lte | neq | gt | lt | eq ) ); -named!(pub dq_string( NomSpan ) -> Token, +named!(pub dq_string( NomSpan ) -> TokenNode, do_parse!( l: position!() >> char!('"') @@ -62,11 +65,11 @@ named!(pub dq_string( NomSpan ) -> Token, >> r1: position!() >> char!('"') >> r: position!() - >> (Spanned::from_nom(RawToken::String(Span::from((l1, r1))), l, r)) + >> (TokenTreeBuilder::spanned_string((l1, r1), (l, r))) ) ); -named!(pub sq_string( NomSpan ) -> Token, +named!(pub sq_string( NomSpan ) -> TokenNode, do_parse!( l: position!() >> char!('\'') @@ -75,61 +78,61 @@ named!(pub sq_string( NomSpan ) -> Token, >> r1: position!() >> char!('\'') >> r: position!() - >> (Spanned::from_nom(RawToken::String(Span::from((l1, r1))), l, r)) + >> (TokenTreeBuilder::spanned_string((l1, r1), (l, r))) ) ); -named!(pub string( NomSpan ) -> Token, +named!(pub string( NomSpan ) -> TokenNode, alt!(sq_string | dq_string) ); -named!(pub bare( NomSpan ) -> Token, +named!(pub bare( NomSpan ) -> TokenNode, do_parse!( l: position!() >> take_while1!(is_start_bare_char) >> take_while!(is_bare_char) >> r: position!() - >> (Spanned::from_nom(RawToken::Bare, l, r)) + >> (TokenTreeBuilder::spanned_bare((l, r))) ) ); -named!(pub var( NomSpan ) -> Token, +named!(pub var( NomSpan ) -> TokenNode, do_parse!( l: position!() >> tag!("$") >> bare: identifier >> r: position!() - >> (Spanned::from_nom(RawToken::Variable(bare.span), l, r)) + >> (TokenTreeBuilder::spanned_var(bare.span(), (l, r))) ) ); -named!(pub identifier( NomSpan ) -> Token, +named!(pub identifier( NomSpan ) -> TokenNode, do_parse!( l: position!() >> take_while1!(is_id_start) >> take_while!(is_id_continue) >> r: position!() - >> (Spanned::from_nom(RawToken::Identifier, l, r)) + >> (TokenTreeBuilder::spanned_ident((l, r))) ) ); -named!(pub flag( NomSpan ) -> Token, +named!(pub flag( NomSpan ) -> TokenNode, do_parse!( l: position!() >> tag!("--") >> bare: bare >> r: position!() - >> (Spanned::from_nom(RawToken::Flag(Flag::Longhand, bare.span), l, r)) + >> (TokenTreeBuilder::spanned_flag(bare.span(), (l, r))) ) ); -named!(pub shorthand( NomSpan ) -> Token, +named!(pub shorthand( NomSpan ) -> TokenNode, do_parse!( l: position!() >> tag!("-") >> bare: bare >> r: position!() - >> (Spanned::from_nom(RawToken::Flag(Flag::Shorthand, bare.span), l, r)) + >> (TokenTreeBuilder::spanned_shorthand(bare.span(), (l, r))) ) ); @@ -142,34 +145,20 @@ named!(pub raw_unit( NomSpan ) -> Spanned, ) ); -named!(pub size( NomSpan ) -> Token, +named!(pub size( NomSpan ) -> TokenNode, do_parse!( l: position!() >> int: raw_integer >> unit: raw_unit >> r: position!() - >> (Spanned::from_nom(RawToken::Size(int.item, unit.item), l, r)) + >> (TokenTreeBuilder::spanned_size((*int, *unit), (l, r))) ) ); -// named!(pub unit_num( NomSpan ) -> Token, -// do_parse!( -// l: position!() -// >> -// ) -// ) - -named!(pub leaf( NomSpan ) -> Token, +named!(pub leaf( NomSpan ) -> TokenNode, alt!(size | integer | string | operator | flag | shorthand | var | bare) ); -named!(pub leaf_node( NomSpan ) -> TokenNode, - do_parse!( - leaf: leaf - >> (TokenNode::Token(leaf)) - ) -); - named!(pub delimited_paren( NomSpan ) -> TokenNode, do_parse!( l: position!() @@ -179,7 +168,30 @@ named!(pub delimited_paren( NomSpan ) -> TokenNode, char!(')') ) >> r: position!() - >> (TokenNode::Delimited(Spanned::from_nom(DelimitedNode::new(Delimiter::Paren, items), l, r))) + >> (TokenTreeBuilder::spanned_parens(items, (l, r))) + ) +); + +named!(pub delimited_brace( NomSpan ) -> TokenNode, + do_parse!( + l: position!() + >> items: delimited!( + char!('{'), + delimited!(space0, separated_list!(space1, node), space0), + char!('}') + ) + >> r: position!() + >> (TokenTreeBuilder::spanned_brace(items, (l, r))) + ) +); + +named!(pub raw_call( NomSpan ) -> TokenNode, + do_parse!( + l: position!() + >> head: node + >> items: opt!(preceded!(space0, separated_nonempty_list!(space1, node))) + >> r: position!() + >> (TokenTreeBuilder::spanned_call((head, items), (l, r))) ) ); @@ -190,16 +202,25 @@ named!(pub path( NomSpan ) -> TokenNode, >> tag!(".") >> tail: separated_list!(tag!("."), alt!(identifier | string)) >> r: position!() - >> (TokenNode::Path(Spanned::from_nom(PathNode::new(Box::new(head), tail.into_iter().map(TokenNode::Token).collect()), l, r))) + >> (TokenTreeBuilder::spanned_path((head, tail), (l, r))) ) ); named!(pub node1( NomSpan ) -> TokenNode, - alt!(leaf_node | delimited_paren) + alt!(leaf | delimited_paren) ); named!(pub node( NomSpan ) -> TokenNode, - alt!(path | leaf_node | delimited_paren) + alt!(path | leaf | delimited_paren | delimited_brace) +); + +named!(pub pipeline( NomSpan ) -> TokenNode, + do_parse!( + l: position!() + >> list: separated_list!(delimited!(space0, tag!("|"), space0), raw_call) + >> r: position!() + >> (TokenTreeBuilder::spanned_pipeline(list, (l, r))) + ) ); fn int(frag: &str, neg: Option) -> i64 { @@ -273,13 +294,13 @@ mod tests { ); assert_eq!( - apply(leaf_node, $input), - TokenNode::Token(token(RawToken::$kind $parens, $left, $right)) + apply(leaf, $input), + token(RawToken::$kind $parens, $left, $right) ); assert_eq!( apply(node, $input), - TokenNode::Token(token(RawToken::$kind $parens, $left, $right)) + token(RawToken::$kind $parens, $left, $right) ); }; @@ -512,6 +533,117 @@ mod tests { ) } + #[test] + fn test_smoke_single_command() { + assert_eq!( + apply(raw_call, "git add ."), + build(b::call( + b::bare("git"), + vec![b::sp(), b::bare("add"), b::sp(), b::bare(".")] + )) + ); + + assert_eq!( + apply(raw_call, "open Cargo.toml"), + build(b::call( + b::bare("open"), + vec![b::sp(), b::bare("Cargo.toml")] + )) + ); + + assert_eq!( + apply(raw_call, "select package.version"), + build(b::call( + b::bare("select"), + vec![b::sp(), b::bare("package.version")] + )) + ); + + assert_eq!( + apply(raw_call, "echo $it"), + build(b::call(b::bare("echo"), vec![b::sp(), b::var("it")])) + ); + + assert_eq!( + apply(raw_call, "open Cargo.toml --raw"), + build(b::call( + b::bare("open"), + vec![b::sp(), b::bare("Cargo.toml"), b::sp(), b::flag("raw")] + )) + ); + + assert_eq!( + apply(raw_call, "open Cargo.toml -r"), + build(b::call( + b::bare("open"), + vec![b::sp(), b::bare("Cargo.toml"), b::sp(), b::shorthand("r")] + )) + ); + + assert_eq!( + apply(raw_call, "config --set tabs 2"), + build(b::call( + b::bare("config"), + vec![ + b::sp(), + b::flag("set"), + b::sp(), + b::bare("tabs"), + b::sp(), + b::int(2) + ] + )) + ); + } + + #[test] + fn test_smoke_pipeline() { + assert_eq!( + apply( + pipeline, + r#"git branch --merged | split-row "`n" | where $it != "* master""# + ), + build(b::pipeline(vec![ + b::call( + b::bare("git"), + vec![b::sp(), b::bare("branch"), b::sp(), b::flag("merged")] + ), + b::call(b::bare("split-row"), vec![b::sp(), b::string("`n")]), + b::call( + b::bare("where"), + vec![ + b::sp(), + b::var("it"), + b::sp(), + b::op("!="), + b::sp(), + b::string("* master") + ] + ) + ])) + ); + + assert_eq!( + apply(pipeline, "ls | where { $it.size > 100 }"), + build(b::pipeline(vec![ + b::call(b::bare("ls"), vec![]), + b::call( + b::bare("where"), + vec![ + b::sp(), + b::braced(vec![ + b::path(b::var("it"), vec![b::ident("size")]), + b::sp(), + b::op(">"), + b::sp(), + b::int(100) + ]) + ] + ) + ])) + ) + } + fn apply(f: impl Fn(NomSpan) -> Result<(NomSpan, T), nom::Err>, string: &str) -> T { match f(NomSpan::new(CompleteStr(string))) { Ok(v) => v.1, @@ -550,8 +682,8 @@ mod tests { TokenNode::Token(Spanned::from_item(token, (left, right))) } - fn token(token: RawToken, left: usize, right: usize) -> Token { - Spanned::from_item(token, (left, right)) + fn token(token: RawToken, left: usize, right: usize) -> TokenNode { + TokenNode::Token(Spanned::from_item(token, (left, right))) } fn build(block: CurriedToken) -> TokenNode { diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index 6ef1e78289..b3cd722016 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -1,20 +1,42 @@ -use crate::parser::parse2::{span::*, tokens::*}; +use crate::parser::parse2::{operator::*, span::*, tokens::*}; use derive_new::new; use enum_utils::FromStr; #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] pub enum TokenNode { Token(Token), + Call(Spanned), Delimited(Spanned), + Pipeline(Spanned>), + Binary(Spanned), Path(Spanned), } +impl TokenNode { + pub fn span(&self) -> Span { + match self { + TokenNode::Token(t) => t.span, + TokenNode::Call(s) => s.span, + TokenNode::Delimited(s) => s.span, + TokenNode::Pipeline(s) => s.span, + TokenNode::Binary(s) => s.span, + TokenNode::Path(s) => s.span, + } + } +} + #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] pub struct DelimitedNode { delimiter: Delimiter, children: Vec, } +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] +pub struct CallNode { + head: Box, + children: Vec, +} + #[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, FromStr)] pub enum Delimiter { Paren, @@ -27,3 +49,10 @@ pub struct PathNode { head: Box, tail: Vec, } + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] +pub struct BinaryNode { + left: Box, + op: Operator, + right: Box, +} diff --git a/src/parser/parse2/token_tree_builder.rs b/src/parser/parse2/token_tree_builder.rs index 5c7df7de77..a3fffaa8d7 100644 --- a/src/parser/parse2/token_tree_builder.rs +++ b/src/parser/parse2/token_tree_builder.rs @@ -1,6 +1,9 @@ +use crate::parser::parse2::flag::Flag; use crate::parser::parse2::operator::Operator; use crate::parser::parse2::span::{Span, Spanned}; -use crate::parser::parse2::token_tree::{DelimitedNode, Delimiter, PathNode, TokenNode}; +use crate::parser::parse2::token_tree::{ + BinaryNode, CallNode, DelimitedNode, Delimiter, PathNode, TokenNode, +}; use crate::parser::parse2::tokens::{RawToken, Token}; use crate::parser::parse2::unit::Unit; use derive_new::new; @@ -20,6 +23,38 @@ impl TokenTreeBuilder { block(&mut builder) } + pub fn pipeline(input: Vec) -> CurriedToken { + Box::new(move |b| { + let start = b.pos; + + let mut out = vec![]; + + let mut input = input.into_iter(); + let first = input + .next() + .expect("A pipeline must contain at least one element"); + + out.push(first(b).expect("The first element of a pipeline must not be whitespace")); + + for item in input { + b.consume(" | "); + + match item(b) { + None => {} + Some(v) => out.push(v), + } + } + + let end = b.pos; + + Some(TokenTreeBuilder::spanned_pipeline(out, (start, end))) + }) + } + + pub fn spanned_pipeline(input: Vec, span: impl Into) -> TokenNode { + TokenNode::Pipeline(Spanned::from_item(input, span)) + } + pub fn op(input: impl Into) -> CurriedToken { let input = input.into(); @@ -164,6 +199,48 @@ impl TokenTreeBuilder { )) } + pub fn flag(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, _) = b.consume("--"); + let (inner_start, end) = b.consume(&input); + + Some(TokenTreeBuilder::spanned_flag( + (inner_start, end), + (start, end), + )) + }) + } + + pub fn spanned_flag(input: impl Into, span: impl Into) -> TokenNode { + TokenNode::Token(Spanned::from_item( + RawToken::Flag(Flag::Longhand, input.into()), + span.into(), + )) + } + + pub fn shorthand(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, _) = b.consume("-"); + let (inner_start, end) = b.consume(&input); + + Some(TokenTreeBuilder::spanned_shorthand( + (inner_start, end), + (start, end), + )) + }) + } + + pub fn spanned_shorthand(input: impl Into, span: impl Into) -> TokenNode { + TokenNode::Token(Spanned::from_item( + RawToken::Flag(Flag::Shorthand, input.into()), + span.into(), + )) + } + pub fn ident(input: impl Into) -> CurriedToken { let input = input.into(); @@ -177,6 +254,39 @@ impl TokenTreeBuilder { TokenNode::Token(Spanned::from_item(RawToken::Identifier, span.into())) } + pub fn call(head: CurriedToken, input: Vec) -> CurriedToken { + Box::new(move |b| { + let start = b.pos; + + let head_node = head(b).expect("The head of a command must not be whitespace"); + + let mut tail_nodes = vec![]; + for item in input { + match item(b) { + None => {} + Some(v) => tail_nodes.push(v), + }; + } + + let end = b.pos; + + Some(TokenTreeBuilder::spanned_call( + (head_node, Some(tail_nodes)), + (start, end), + )) + }) + } + + pub fn spanned_call( + input: (TokenNode, Option>), + span: impl Into, + ) -> TokenNode { + TokenNode::Call(Spanned::from_item( + CallNode::new(Box::new(input.0), input.1.unwrap_or_else(|| vec![])), + span, + )) + } + pub fn parens(input: Vec) -> CurriedToken { Box::new(move |b| { let (start, _) = b.consume("("); @@ -190,13 +300,81 @@ impl TokenTreeBuilder { let (_, end) = b.consume(")"); - Some(TokenNode::Delimited(Spanned::from_item( - DelimitedNode::new(Delimiter::Paren, output), - (start, end), - ))) + Some(TokenTreeBuilder::spanned_parens(output, (start, end))) }) } + pub fn spanned_parens(input: impl Into>, span: impl Into) -> TokenNode { + TokenNode::Delimited(Spanned::from_item( + DelimitedNode::new(Delimiter::Paren, input.into()), + span, + )) + } + + pub fn braced(input: Vec) -> CurriedToken { + Box::new(move |b| { + let (start, _) = b.consume("{ "); + let mut output = vec![]; + for item in input { + match item(b) { + None => {} + Some(v) => output.push(v), + }; + } + + let (_, end) = b.consume(" }"); + + Some(TokenTreeBuilder::spanned_brace(output, (start, end))) + }) + } + + pub fn spanned_brace(input: impl Into>, span: impl Into) -> TokenNode { + TokenNode::Delimited(Spanned::from_item( + DelimitedNode::new(Delimiter::Brace, input.into()), + span, + )) + } + + pub fn binary( + left: CurriedToken, + op: impl Into, + right: CurriedToken, + ) -> CurriedToken { + let op = op.into(); + + Box::new(move |b| { + let start = b.pos; + + let left = left(b).expect("The left side of a binary operation must not be whitespace"); + + b.consume(" "); + + b.consume(op.as_str()); + + b.consume(" "); + + let right = + right(b).expect("The right side of a binary operation must not be whitespace"); + + let end = b.pos; + + Some(TokenTreeBuilder::spanned_binary( + (left, op, right), + (start, end), + )) + }) + } + + pub fn spanned_binary( + input: (impl Into, Operator, impl Into), + span: impl Into, + ) -> TokenNode { + TokenNode::Binary(Spanned::from_item( + BinaryNode::new(Box::new(input.0.into()), input.1, Box::new(input.2.into())), + span, + )) + } + pub fn sp() -> CurriedToken { Box::new(|b| { b.consume(" ");