From 77bf7612035cdec0d2a7b27e54087ed6cb19f9e8 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 2 Oct 2021 20:11:08 +0300 Subject: [PATCH 1/3] internal: move code to where it's used and reduce visibility --- crates/mbe/src/expander/matcher.rs | 60 +++++++++++++++++++- crates/mbe/src/parser.rs | 88 +++++------------------------- 2 files changed, 73 insertions(+), 75 deletions(-) diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 7d8f97c775..233ca08dc1 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -63,7 +63,7 @@ use std::rc::Rc; use crate::{ expander::{Binding, Bindings, Fragment}, - parser::{Op, OpDelimited, OpDelimitedIter, RepeatKind, Separator}, + parser::{Op, RepeatKind, Separator}, tt_iter::TtIter, ExpandError, MetaTemplate, }; @@ -750,6 +750,64 @@ fn collect_vars(buf: &mut Vec, pattern: &MetaTemplate) { } } +impl MetaTemplate { + fn iter_delimited<'a>(&'a self, delimited: Option<&'a tt::Delimiter>) -> OpDelimitedIter<'a> { + OpDelimitedIter { inner: &self.0, idx: 0, delimited } + } +} + +#[derive(Debug, Clone, Copy)] +enum OpDelimited<'a> { + Op(&'a Op), + Open, + Close, +} + +#[derive(Debug, Clone, Copy)] +struct OpDelimitedIter<'a> { + inner: &'a Vec, + delimited: Option<&'a tt::Delimiter>, + idx: usize, +} + +impl<'a> OpDelimitedIter<'a> { + fn is_eof(&self) -> bool { + let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 }; + self.idx >= len + } + + fn peek(&self) -> Option> { + match self.delimited { + None => self.inner.get(self.idx).map(OpDelimited::Op), + Some(_) => match self.idx { + 0 => Some(OpDelimited::Open), + i if i == self.inner.len() + 1 => Some(OpDelimited::Close), + i => self.inner.get(i - 1).map(OpDelimited::Op), + }, + } + } + + fn reset(&self) -> Self { + Self { inner: self.inner, idx: 0, delimited: self.delimited } + } +} + +impl<'a> Iterator for OpDelimitedIter<'a> { + type Item = OpDelimited<'a>; + + fn next(&mut self) -> Option { + let res = self.peek(); + self.idx += 1; + res + } + + fn size_hint(&self) -> (usize, Option) { + let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 }; + let remain = len.saturating_sub(self.idx); + (remain, Some(remain)) + } +} + impl<'a> TtIter<'a> { fn expect_separator(&mut self, separator: &Separator, idx: usize) -> bool { let mut fork = self.clone(); diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs index deed884d2d..0cce4146fb 100644 --- a/crates/mbe/src/parser.rs +++ b/crates/mbe/src/parser.rs @@ -3,76 +3,15 @@ use smallvec::SmallVec; use syntax::SmolStr; -use tt::Delimiter; use crate::{tt_iter::TtIter, ParseError}; -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct MetaTemplate(pub(crate) Vec); - -#[derive(Debug, Clone, Copy)] -pub(crate) enum OpDelimited<'a> { - Op(&'a Op), - Open, - Close, +pub(crate) fn parse_template(template: &tt::Subtree) -> Result, ParseError> { + parse_inner(template, Mode::Template).into_iter().collect() } -#[derive(Debug, Clone, Copy)] -pub(crate) struct OpDelimitedIter<'a> { - inner: &'a Vec, - delimited: Option<&'a Delimiter>, - idx: usize, -} - -impl<'a> OpDelimitedIter<'a> { - pub(crate) fn is_eof(&self) -> bool { - let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 }; - self.idx >= len - } - - pub(crate) fn peek(&self) -> Option> { - match self.delimited { - None => self.inner.get(self.idx).map(OpDelimited::Op), - Some(_) => match self.idx { - 0 => Some(OpDelimited::Open), - i if i == self.inner.len() + 1 => Some(OpDelimited::Close), - i => self.inner.get(i - 1).map(OpDelimited::Op), - }, - } - } - - pub(crate) fn reset(&self) -> Self { - Self { inner: self.inner, idx: 0, delimited: self.delimited } - } -} - -impl<'a> Iterator for OpDelimitedIter<'a> { - type Item = OpDelimited<'a>; - - fn next(&mut self) -> Option { - let res = self.peek(); - self.idx += 1; - res - } - - fn size_hint(&self) -> (usize, Option) { - let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 }; - let remain = len.saturating_sub(self.idx); - (remain, Some(remain)) - } -} - -impl<'a> MetaTemplate { - pub(crate) fn iter(&self) -> impl Iterator { - self.0.iter() - } - - pub(crate) fn iter_delimited( - &'a self, - delimited: Option<&'a Delimiter>, - ) -> OpDelimitedIter<'a> { - OpDelimitedIter { inner: &self.0, idx: 0, delimited } - } +pub(crate) fn parse_pattern(pattern: &tt::Subtree) -> Result, ParseError> { + parse_inner(pattern, Mode::Pattern).into_iter().collect() } #[derive(Clone, Debug, PartialEq, Eq)] @@ -80,7 +19,7 @@ pub(crate) enum Op { Var { name: SmolStr, kind: Option, id: tt::TokenId }, Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option }, Leaf(tt::Leaf), - Subtree { tokens: MetaTemplate, delimiter: Option }, + Subtree { tokens: MetaTemplate, delimiter: Option }, } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -97,6 +36,15 @@ pub(crate) enum Separator { Puncts(SmallVec<[tt::Punct; 3]>), } +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct MetaTemplate(pub(crate) Vec); + +impl MetaTemplate { + pub(crate) fn iter(&self) -> impl Iterator { + self.0.iter() + } +} + // Note that when we compare a Separator, we just care about its textual value. impl PartialEq for Separator { fn eq(&self, other: &Separator) -> bool { @@ -125,14 +73,6 @@ impl Separator { } } -pub(crate) fn parse_template(template: &tt::Subtree) -> Result, ParseError> { - parse_inner(template, Mode::Template).into_iter().collect() -} - -pub(crate) fn parse_pattern(pattern: &tt::Subtree) -> Result, ParseError> { - parse_inner(pattern, Mode::Pattern).into_iter().collect() -} - #[derive(Clone, Copy)] enum Mode { Pattern, From 613609cc5eb31a87c08b63b8d5a5ac8123a7b84a Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 2 Oct 2021 20:21:23 +0300 Subject: [PATCH 2/3] minor: cleanup --- crates/mbe/src/expander/matcher.rs | 2 +- crates/mbe/src/lib.rs | 6 ++-- crates/mbe/src/parser.rs | 45 ++++++++++++++++++++---------- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 233ca08dc1..06c21dfd70 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -765,7 +765,7 @@ enum OpDelimited<'a> { #[derive(Debug, Clone, Copy)] struct OpDelimitedIter<'a> { - inner: &'a Vec, + inner: &'a [Op], delimited: Option<&'a tt::Delimiter>, idx: usize, } diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 0a1b7cc848..8470ea0aaf 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -19,7 +19,7 @@ mod token_map; use std::fmt; use crate::{ - parser::{parse_pattern, parse_template, MetaTemplate, Op}, + parser::{MetaTemplate, Op}, tt_iter::TtIter, }; @@ -275,8 +275,8 @@ impl Rule { .expect_subtree() .map_err(|()| ParseError::Expected("expected subtree".to_string()))?; - let lhs = MetaTemplate(parse_pattern(lhs)?); - let rhs = MetaTemplate(parse_template(rhs)?); + let lhs = MetaTemplate::parse_pattern(lhs)?; + let rhs = MetaTemplate::parse_template(rhs)?; Ok(crate::Rule { lhs, rhs }) } diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs index 0cce4146fb..f0a9dd4a51 100644 --- a/crates/mbe/src/parser.rs +++ b/crates/mbe/src/parser.rs @@ -6,12 +6,38 @@ use syntax::SmolStr; use crate::{tt_iter::TtIter, ParseError}; -pub(crate) fn parse_template(template: &tt::Subtree) -> Result, ParseError> { - parse_inner(template, Mode::Template).into_iter().collect() -} +/// Consider +/// +/// ``` +/// macro_rules! an_macro { +/// ($x:expr + $y:expr) => ($y * $x) +/// } +/// ``` +/// +/// Stuff to the left of `=>` is a [`MetaTemplate`] pattern (which is matched +/// with input). +/// +/// Stuff to the right is a [`MetaTemplate`] template which is used to produce +/// output. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct MetaTemplate(pub(crate) Vec); -pub(crate) fn parse_pattern(pattern: &tt::Subtree) -> Result, ParseError> { - parse_inner(pattern, Mode::Pattern).into_iter().collect() +impl MetaTemplate { + pub(crate) fn parse_pattern(pattern: &tt::Subtree) -> Result { + let ops = + parse_inner(pattern, Mode::Pattern).into_iter().collect::>()?; + Ok(MetaTemplate(ops)) + } + + pub(crate) fn parse_template(template: &tt::Subtree) -> Result { + let ops = + parse_inner(template, Mode::Template).into_iter().collect::>()?; + Ok(MetaTemplate(ops)) + } + + pub(crate) fn iter(&self) -> impl Iterator { + self.0.iter() + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -36,15 +62,6 @@ pub(crate) enum Separator { Puncts(SmallVec<[tt::Punct; 3]>), } -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct MetaTemplate(pub(crate) Vec); - -impl MetaTemplate { - pub(crate) fn iter(&self) -> impl Iterator { - self.0.iter() - } -} - // Note that when we compare a Separator, we just care about its textual value. impl PartialEq for Separator { fn eq(&self, other: &Separator) -> bool { From 4e352275d17dcba44052f232940223cbbb70c7ce Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 2 Oct 2021 20:38:28 +0300 Subject: [PATCH 3/3] minor: simplify --- crates/mbe/src/parser.rs | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs index f0a9dd4a51..4ce818f9b0 100644 --- a/crates/mbe/src/parser.rs +++ b/crates/mbe/src/parser.rs @@ -24,20 +24,28 @@ pub(crate) struct MetaTemplate(pub(crate) Vec); impl MetaTemplate { pub(crate) fn parse_pattern(pattern: &tt::Subtree) -> Result { - let ops = - parse_inner(pattern, Mode::Pattern).into_iter().collect::>()?; - Ok(MetaTemplate(ops)) + MetaTemplate::parse(pattern, Mode::Pattern) } pub(crate) fn parse_template(template: &tt::Subtree) -> Result { - let ops = - parse_inner(template, Mode::Template).into_iter().collect::>()?; - Ok(MetaTemplate(ops)) + MetaTemplate::parse(template, Mode::Template) } pub(crate) fn iter(&self) -> impl Iterator { self.0.iter() } + + fn parse(tt: &tt::Subtree, mode: Mode) -> Result { + let mut src = TtIter::new(tt); + + let mut res = Vec::new(); + while let Some(first) = src.next() { + let op = next_op(first, &mut src, mode)?; + res.push(op) + } + + Ok(MetaTemplate(res)) + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -96,15 +104,6 @@ enum Mode { Template, } -fn parse_inner(tt: &tt::Subtree, mode: Mode) -> Vec> { - let mut src = TtIter::new(tt); - std::iter::from_fn(move || { - let first = src.next()?; - Some(next_op(first, &mut src, mode)) - }) - .collect() -} - macro_rules! err { ($($tt:tt)*) => { ParseError::UnexpectedToken(($($tt)*).to_string()) @@ -128,10 +127,8 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul match second { tt::TokenTree::Subtree(subtree) => { let (separator, kind) = parse_repeat(src)?; - let tokens = parse_inner(subtree, mode) - .into_iter() - .collect::, ParseError>>()?; - Op::Repeat { tokens: MetaTemplate(tokens), separator, kind } + let tokens = MetaTemplate::parse(subtree, mode)?; + Op::Repeat { tokens, separator, kind } } tt::TokenTree::Leaf(leaf) => match leaf { tt::Leaf::Punct(_) => { @@ -162,9 +159,8 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul } tt::TokenTree::Leaf(tt) => Op::Leaf(tt.clone()), tt::TokenTree::Subtree(subtree) => { - let tokens = - parse_inner(subtree, mode).into_iter().collect::, ParseError>>()?; - Op::Subtree { tokens: MetaTemplate(tokens), delimiter: subtree.delimiter } + let tokens = MetaTemplate::parse(subtree, mode)?; + Op::Subtree { tokens, delimiter: subtree.delimiter } } }; Ok(res)