Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-11-13 08:27:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

use usual token tree for macro expansion

Browse source
This commit is contained in:
Aleksey Kladov 2019-09-17 02:54:22 +03:00
parent 37ef8927c3
commit 4551182f94
10 changed files with 774 additions and 954 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

161
crates/ra_mbe/src/lib.rs
View file

@ -3,30 +3,19 @@
/// interface, although it contains some code to bridge `SyntaxNode`s and
/// `TokenTree`s as well!
macro_rules! impl_froms {
($e:ident: $($v:ident), *) => {
$(
impl From<$v> for $e {
fn from(it: $v) -> $e {
$e::$v(it)
}
}
)*
}
}
mod mbe_parser;
mod parser;
mod mbe_expander;
mod syntax_bridge;
mod tt_cursor;
mod tt_iter;
mod subtree_source;
mod subtree_parser;
use ra_syntax::SmolStr;
use smallvec::SmallVec;
pub use tt::{Delimiter, Punct};
use crate::{
parser::{parse_pattern, Op},
tt_iter::TtIter,
};
#[derive(Debug, PartialEq, Eq)]
pub enum ParseError {
Expected(String),
@ -38,6 +27,7 @@ pub enum ExpandError {
UnexpectedToken,
BindingError(String),
ConversionError,
InvalidRepeat,
}
pub use crate::syntax_bridge::{
@ -54,97 +44,72 @@ pub struct MacroRules {
pub(crate) rules: Vec<Rule>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Rule {
pub(crate) lhs: tt::Subtree,
pub(crate) rhs: tt::Subtree,
}
impl MacroRules {
pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> {
mbe_parser::parse(tt)
let mut src = TtIter::new(tt);
let mut rules = Vec::new();
while src.len() > 0 {
let rule = Rule::parse(&mut src)?;
rules.push(rule);
if let Err(()) = src.expect_char(';') {
if src.len() > 0 {
return Err(ParseError::Expected("expected `:`".to_string()));
}
break;
}
}
Ok(MacroRules { rules })
}
pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
mbe_expander::expand(self, tt)
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Rule {
pub(crate) lhs: Subtree,
pub(crate) rhs: Subtree,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum TokenTree {
Leaf(Leaf),
Subtree(Subtree),
Repeat(Repeat),
}
impl_froms!(TokenTree: Leaf, Subtree, Repeat);
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum Leaf {
Literal(Literal),
Punct(Punct),
Ident(Ident),
Var(Var),
}
impl_froms!(Leaf: Literal, Punct, Ident, Var);
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Subtree {
pub(crate) delimiter: Delimiter,
pub(crate) token_trees: Vec<TokenTree>,
}
#[derive(Clone, Debug, Eq)]
pub(crate) enum Separator {
Literal(tt::Literal),
Ident(tt::Ident),
Puncts(SmallVec<[tt::Punct; 3]>),
}
// Note that when we compare a Separator, we just care about its textual value.
impl PartialEq for crate::Separator {
fn eq(&self, other: &crate::Separator) -> bool {
use crate::Separator::*;
match (self, other) {
(Ident(ref a), Ident(ref b)) => a.text == b.text,
(Literal(ref a), Literal(ref b)) => a.text == b.text,
(Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => {
let a_iter = a.iter().map(|a| a.char);
let b_iter = b.iter().map(|b| b.char);
a_iter.eq(b_iter)
}
_ => false,
}
impl Rule {
fn parse(src: &mut TtIter) -> Result<Rule, ParseError> {
let mut lhs = src
.expect_subtree()
.map_err(|()| ParseError::Expected("expected subtree".to_string()))?
.clone();
validate(&lhs)?;
lhs.delimiter = tt::Delimiter::None;
src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
let mut rhs = src
.expect_subtree()
.map_err(|()| ParseError::Expected("expected subtree".to_string()))?
.clone();
rhs.delimiter = tt::Delimiter::None;
Ok(crate::Rule { lhs, rhs })
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Repeat {
pub(crate) subtree: Subtree,
pub(crate) kind: RepeatKind,
pub(crate) separator: Option<Separator>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum RepeatKind {
ZeroOrMore,
OneOrMore,
ZeroOrOne,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Literal {
pub(crate) text: SmolStr,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Ident {
pub(crate) text: SmolStr,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Var {
pub(crate) text: SmolStr,
pub(crate) kind: Option<SmolStr>,
fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> {
for op in parse_pattern(pattern) {
let op = match op {
Ok(it) => it,
Err(e) => {
let msg = match e {
ExpandError::InvalidRepeat => "invalid repeat".to_string(),
_ => "invalid macro definition".to_string(),
};
return Err(ParseError::Expected(msg));
}
};
match op {
Op::TokenTree(tt::TokenTree::Subtree(subtree)) | Op::Repeat { subtree, .. } => {
validate(subtree)?
}
_ => (),
}
}
Ok(())
}
#[cfg(test)]

19
crates/ra_mbe/src/mbe_expander.rs
View file

@ -8,7 +8,6 @@ mod transcriber;
use ra_syntax::SmolStr;
use rustc_hash::FxHashMap;
use crate::tt_cursor::TtCursor;
use crate::ExpandError;
pub(crate) fn expand(
@ -19,12 +18,8 @@ pub(crate) fn expand(
}
fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
let mut input = TtCursor::new(input);
let bindings = matcher::match_lhs(&rule.lhs, &mut input)?;
if !input.is_eof() {
return Err(ExpandError::UnexpectedToken);
}
let res = transcriber::transcribe(&bindings, &rule.rhs)?;
let bindings = matcher::match_(&rule.lhs, input)?;
let res = transcriber::transcribe(&rule.rhs, &bindings)?;
Ok(res)
}
@ -103,13 +98,6 @@ mod tests {
#[test]
fn test_expand_rule() {
// FIXME: The missing $var check should be in parsing phase
// assert_err(
// "($i:ident) => ($j)",
// "foo!{a}",
// ExpandError::BindingError(String::from("could not find binding `j`")),
// );
assert_err(
"($($i:ident);*) => ($i)",
"foo!{a}",
@ -118,9 +106,6 @@ mod tests {
)),
);
assert_err("($i) => ($i)", "foo!{a}", ExpandError::UnexpectedToken);
assert_err("($i:) => ($i)", "foo!{a}", ExpandError::UnexpectedToken);
// FIXME:
// Add an err test case for ($($i:ident)) => ($())
}

387
crates/ra_mbe/src/mbe_expander/matcher.rs
View file

@ -1,11 +1,14 @@
use crate::{
mbe_expander::{Binding, Bindings, Fragment},
tt_cursor::TtCursor,
parser::{parse_pattern, Op, RepeatKind, Separator},
subtree_source::SubtreeTokenSource,
tt_iter::TtIter,
ExpandError,
};
use ra_parser::FragmentKind::*;
use ra_syntax::SmolStr;
use ra_parser::{FragmentKind::*, TreeSink};
use ra_syntax::{SmolStr, SyntaxKind};
use tt::buffer::{Cursor, TokenBuffer};
impl Bindings {
fn push_optional(&mut self, name: &SmolStr) {
@ -42,121 +45,247 @@ impl Bindings {
}
Ok(())
}
fn merge(&mut self, nested: Bindings) {
self.inner.extend(nested.inner);
}
}
pub(super) fn match_lhs(
pattern: &crate::Subtree,
input: &mut TtCursor,
) -> Result<Bindings, ExpandError> {
macro_rules! err {
() => {
ExpandError::BindingError(format!(""))
};
($($tt:tt)*) => {
ExpandError::BindingError(format!($($tt)*))
};
}
macro_rules! bail {
($($tt:tt)*) => {
return Err(err!($($tt)*))
};
}
pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result<Bindings, ExpandError> {
assert!(pattern.delimiter == tt::Delimiter::None);
let mut res = Bindings::default();
for pat in pattern.token_trees.iter() {
match pat {
crate::TokenTree::Leaf(leaf) => match leaf {
crate::Leaf::Var(crate::Var { text, kind }) => {
let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?;
match match_meta_var(kind.as_str(), input)? {
Some(fragment) => {
res.inner.insert(text.clone(), Binding::Fragment(fragment));
}
None => res.push_optional(text),
}
}
crate::Leaf::Punct(punct) => {
if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) {
return Err(ExpandError::UnexpectedToken);
}
}
crate::Leaf::Ident(ident) => {
if input.eat_ident().map(|i| &i.text) != Some(&ident.text) {
return Err(ExpandError::UnexpectedToken);
}
}
crate::Leaf::Literal(literal) => {
if input.eat_literal().map(|i| &i.text) != Some(&literal.text) {
return Err(ExpandError::UnexpectedToken);
}
}
},
crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => {
// Dirty hack to make macro-expansion terminate.
// This should be replaced by a propper macro-by-example implementation
let mut limit = 65536;
let mut counter = 0;
let mut src = TtIter::new(src);
let mut memento = input.save();
match_subtree(&mut res, pattern, &mut src)?;
loop {
match match_lhs(subtree, input) {
Ok(nested) => {
limit -= 1;
if limit == 0 {
log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator);
break;
}
memento = input.save();
res.push_nested(counter, nested)?;
counter += 1;
if counter == 1 {
if let crate::RepeatKind::ZeroOrOne = kind {
break;
}
}
if let Some(separator) = separator {
if !input
.eat_seperator()
.map(|sep| sep == *separator)
.unwrap_or(false)
{
input.rollback(memento);
break;
}
}
}
Err(_) => {
input.rollback(memento);
break;
}
}
}
match kind {
crate::RepeatKind::OneOrMore if counter == 0 => {
return Err(ExpandError::UnexpectedToken);
}
_ if counter == 0 => {
// Collect all empty variables in subtrees
collect_vars(subtree).iter().for_each(|s| res.push_empty(s));
}
_ => {}
}
}
crate::TokenTree::Subtree(subtree) => {
let input_subtree =
input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?;
if subtree.delimiter != input_subtree.delimiter {
return Err(ExpandError::UnexpectedToken);
}
let mut input = TtCursor::new(input_subtree);
let bindings = match_lhs(&subtree, &mut input)?;
if !input.is_eof() {
return Err(ExpandError::UnexpectedToken);
}
res.merge(bindings);
}
}
if src.len() > 0 {
bail!("leftover tokens");
}
Ok(res)
}
fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>, ExpandError> {
fn match_subtree(
bindings: &mut Bindings,
pattern: &tt::Subtree,
src: &mut TtIter,
) -> Result<(), ExpandError> {
for op in parse_pattern(pattern) {
match op? {
Op::TokenTree(tt::TokenTree::Leaf(lhs)) => {
let rhs = src.expect_leaf().map_err(|()| err!("expected leaf: `{}`", lhs))?;
match (lhs, rhs) {
(
tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
) if lhs == rhs => (),
(
tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
) if lhs == rhs => (),
(
tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
) if lhs == rhs => (),
_ => Err(ExpandError::UnexpectedToken)?,
}
}
Op::TokenTree(tt::TokenTree::Subtree(lhs)) => {
let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?;
if lhs.delimiter != rhs.delimiter {
bail!("mismatched delimiter")
}
let mut src = TtIter::new(rhs);
match_subtree(bindings, lhs, &mut src)?;
if src.len() > 0 {
bail!("leftover tokens");
}
}
Op::Var { name, kind } => {
let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?;
match match_meta_var(kind.as_str(), src)? {
Some(fragment) => {
bindings.inner.insert(name.clone(), Binding::Fragment(fragment));
}
None => bindings.push_optional(name),
}
()
}
Op::Repeat { subtree, kind, separator } => {
match_repeat(bindings, subtree, kind, separator, src)?
}
}
}
Ok(())
}
impl<'a> TtIter<'a> {
fn eat_separator(&mut self, separator: &Separator) -> bool {
let mut fork = self.clone();
let ok = match separator {
Separator::Ident(lhs) => match fork.expect_ident() {
Ok(rhs) => rhs.text == lhs.text,
_ => false,
},
Separator::Literal(lhs) => match fork.expect_literal() {
Ok(rhs) => rhs.text == lhs.text,
_ => false,
},
Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() {
Ok(rhs) => rhs.char == lhs.char,
_ => false,
}),
};
if ok {
*self = fork;
}
ok
}
pub(crate) fn expect_lifetime(&mut self) -> Result<&tt::Ident, ()> {
let ident = self.expect_ident()?;
// check if it start from "`"
if ident.text.chars().next() != Some('\'') {
return Err(());
}
Ok(ident)
}
pub(crate) fn expect_fragment(
&mut self,
fragment_kind: ra_parser::FragmentKind,
) -> Result<tt::TokenTree, ()> {
pub(crate) struct OffsetTokenSink<'a> {
pub(crate) cursor: Cursor<'a>,
pub(crate) error: bool,
}
impl<'a> TreeSink for OffsetTokenSink<'a> {
fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) {
for _ in 0..n_tokens {
self.cursor = self.cursor.bump_subtree();
}
}
fn start_node(&mut self, _kind: SyntaxKind) {}
fn finish_node(&mut self) {}
fn error(&mut self, _error: ra_parser::ParseError) {
self.error = true;
}
}
let buffer = TokenBuffer::new(self.inner.as_slice());
let mut src = SubtreeTokenSource::new(&buffer);
let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
ra_parser::parse_fragment(&mut src, &mut sink, fragment_kind);
if !sink.cursor.is_root() || sink.error {
return Err(());
}
let mut curr = buffer.begin();
let mut res = vec![];
while curr != sink.cursor {
if let Some(token) = curr.token_tree() {
res.push(token);
}
curr = curr.bump();
}
self.inner = self.inner.as_slice()[res.len()..].iter();
match res.len() {
0 => Err(()),
1 => Ok(res[0].clone()),
_ => Ok(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::None,
token_trees: res.into_iter().cloned().collect(),
})),
}
}
pub(crate) fn eat_vis(&mut self) -> Option<tt::TokenTree> {
let mut fork = self.clone();
match fork.expect_fragment(Visibility) {
Ok(tt) => {
*self = fork;
Some(tt)
}
Err(()) => None,
}
}
}
pub(super) fn match_repeat(
bindings: &mut Bindings,
pattern: &tt::Subtree,
kind: RepeatKind,
separator: Option<Separator>,
src: &mut TtIter,
) -> Result<(), ExpandError> {
// Dirty hack to make macro-expansion terminate.
// This should be replaced by a propper macro-by-example implementation
let mut limit = 65536;
let mut counter = 0;
for i in 0.. {
let mut fork = src.clone();
if let Some(separator) = &separator {
if i != 0 && !fork.eat_separator(separator) {
break;
}
}
let mut nested = Bindings::default();
match match_subtree(&mut nested, pattern, &mut fork) {
Ok(()) => {
limit -= 1;
if limit == 0 {
log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", pattern, src, kind, separator);
break;
}
*src = fork;
bindings.push_nested(counter, nested)?;
counter += 1;
if counter == 1 {
if let RepeatKind::ZeroOrOne = kind {
break;
}
}
}
Err(_) => break,
}
}
match (kind, counter) {
(RepeatKind::OneOrMore, 0) => return Err(ExpandError::UnexpectedToken),
(_, 0) => {
// Collect all empty variables in subtrees
let mut vars = Vec::new();
collect_vars(&mut vars, pattern)?;
for var in vars {
bindings.push_empty(&var)
}
}
_ => (),
}
Ok(())
}
fn match_meta_var(kind: &str, input: &mut TtIter) -> Result<Option<Fragment>, ExpandError> {
let fragment = match kind {
"path" => Path,
"expr" => Expr,
@ -169,17 +298,20 @@ fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>,
_ => {
let tt = match kind {
"ident" => {
let ident = input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone();
let ident = input.expect_ident().map_err(|()| err!("expected ident"))?.clone();
tt::Leaf::from(ident).into()
}
"tt" => input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(),
"lifetime" => input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone(),
"tt" => input.next().ok_or_else(|| err!())?.clone(),
"lifetime" => {
let ident = input.expect_lifetime().map_err(|()| err!())?;
tt::Leaf::Ident(ident.clone()).into()
}
"literal" => {
let literal = input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone();
let literal = input.expect_literal().map_err(|()| err!())?.clone();
tt::Leaf::from(literal).into()
}
// `vis` is optional
"vis" => match input.try_eat_vis() {
"vis" => match input.eat_vis() {
Some(vis) => vis,
None => return Ok(None),
},
@ -188,28 +320,19 @@ fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>,
return Ok(Some(Fragment::Tokens(tt)));
}
};
let tt = input.eat_fragment(fragment).ok_or(ExpandError::UnexpectedToken)?;
let tt = input.expect_fragment(fragment).map_err(|()| err!())?;
let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) };
Ok(Some(fragment))
}
fn collect_vars(subtree: &crate::Subtree) -> Vec<SmolStr> {
let mut res = Vec::new();
for tkn in subtree.token_trees.iter() {
match tkn {
crate::TokenTree::Leaf(crate::Leaf::Var(crate::Var { text, .. })) => {
res.push(text.clone());
}
crate::TokenTree::Subtree(subtree) => {
res.extend(collect_vars(subtree));
}
crate::TokenTree::Repeat(crate::Repeat { subtree, .. }) => {
res.extend(collect_vars(subtree));
}
_ => {}
fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &tt::Subtree) -> Result<(), ExpandError> {
for op in parse_pattern(pattern) {
match op? {
Op::Var { name, .. } => buf.push(name.clone()),
Op::TokenTree(tt::TokenTree::Leaf(_)) => (),
Op::TokenTree(tt::TokenTree::Subtree(subtree)) => collect_vars(buf, subtree)?,
Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?,
}
}
res
Ok(())
}

300
crates/ra_mbe/src/mbe_expander/transcriber.rs
View file

@ -1,16 +1,20 @@
//! Transcraber takes a template, like `fn $ident() {}`, a set of bindings like
//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
use ra_syntax::SmolStr;
use crate::{
mbe_expander::{Binding, Bindings, Fragment},
parser::{parse_template, Op, RepeatKind, Separator},
ExpandError,
};
impl Bindings {
fn contains(&self, name: &SmolStr) -> bool {
fn contains(&self, name: &str) -> bool {
self.inner.contains_key(name)
}
fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> {
fn get(&self, name: &str, nesting: &[usize]) -> Result<&Fragment, ExpandError> {
let mut b = self.inner.get(name).ok_or_else(|| {
ExpandError::BindingError(format!("could not find binding `{}`", name))
})?;
@ -43,11 +47,12 @@ impl Bindings {
}
pub(super) fn transcribe(
template: &tt::Subtree,
bindings: &Bindings,
template: &crate::Subtree,
) -> Result<tt::Subtree, ExpandError> {
assert!(template.delimiter == tt::Delimiter::None);
let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false };
expand_subtree(template, &mut ctx)
expand_subtree(&mut ctx, template)
}
#[derive(Debug)]
@ -57,159 +62,158 @@ struct ExpandCtx<'a> {
var_expanded: bool,
}
fn expand_subtree(
template: &crate::Subtree,
ctx: &mut ExpandCtx,
) -> Result<tt::Subtree, ExpandError> {
fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
let mut buf: Vec<tt::TokenTree> = Vec::new();
for tt in template.token_trees.iter() {
let tt = expand_tt(tt, ctx)?;
push_fragment(&mut buf, tt);
for op in parse_template(template) {
match op? {
Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => buf.push(tt.clone()),
Op::TokenTree(tt::TokenTree::Subtree(tt)) => {
let tt = expand_subtree(ctx, tt)?;
buf.push(tt.into());
}
Op::Var { name, kind: _ } => {
let fragment = expand_var(ctx, name)?;
push_fragment(&mut buf, fragment);
}
Op::Repeat { subtree, kind, separator } => {
let fragment = expand_repeat(ctx, subtree, kind, separator)?;
push_fragment(&mut buf, fragment)
}
}
}
Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf })
}
fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result<Fragment, ExpandError> {
let res: tt::TokenTree = match template {
crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(),
crate::TokenTree::Repeat(repeat) => {
let mut buf: Vec<tt::TokenTree> = Vec::new();
ctx.nesting.push(0);
// Dirty hack to make macro-expansion terminate.
// This should be replaced by a propper macro-by-example implementation
let mut limit = 65536;
let mut has_seps = 0;
let mut counter = 0;
// We store the old var expanded value, and restore it later
// It is because before this `$repeat`,
// it is possible some variables already expanad in the same subtree
//
// `some_var_expanded` keep check if the deeper subtree has expanded variables
let mut some_var_expanded = false;
let old_var_expanded = ctx.var_expanded;
ctx.var_expanded = false;
while let Ok(t) = expand_subtree(&repeat.subtree, ctx) {
// if no var expanded in the child, we count it as a fail
if !ctx.var_expanded {
break;
}
// Reset `ctx.var_expandeded` to see if there is other expanded variable
// in the next matching
some_var_expanded = true;
ctx.var_expanded = false;
counter += 1;
limit -= 1;
if limit == 0 {
log::warn!(
"expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
template,
ctx
);
break;
}
let idx = ctx.nesting.pop().unwrap();
ctx.nesting.push(idx + 1);
push_subtree(&mut buf, t);
if let Some(ref sep) = repeat.separator {
match sep {
crate::Separator::Ident(ident) => {
has_seps = 1;
buf.push(tt::Leaf::from(ident.clone()).into());
}
crate::Separator::Literal(lit) => {
has_seps = 1;
buf.push(tt::Leaf::from(lit.clone()).into());
}
crate::Separator::Puncts(puncts) => {
has_seps = puncts.len();
for punct in puncts {
buf.push(tt::Leaf::from(*punct).into());
}
}
}
}
if let crate::RepeatKind::ZeroOrOne = repeat.kind {
break;
}
}
// Restore the `var_expanded` by combining old one and the new one
ctx.var_expanded = some_var_expanded || old_var_expanded;
ctx.nesting.pop().unwrap();
for _ in 0..has_seps {
buf.pop();
}
if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 {
return Err(ExpandError::UnexpectedToken);
}
// Check if it is a single token subtree without any delimiter
// e.g {Delimiter:None> ['>'] /Delimiter:None>}
tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into()
fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result<Fragment, ExpandError> {
let res = if v == "crate" {
// FIXME: Properly handle $crate token
let tt =
tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() })
.into();
Fragment::Tokens(tt)
} else if !ctx.bindings.contains(v) {
// Note that it is possible to have a `$var` inside a macro which is not bound.
// For example:
// ```
// macro_rules! foo {
// ($a:ident, $b:ident, $c:tt) => {
// macro_rules! bar {
// ($bi:ident) => {
// fn $bi() -> u8 {$c}
// }
// }
// }
// ```
// We just treat it a normal tokens
let tt = tt::Subtree {
delimiter: tt::Delimiter::None,
token_trees: vec![
tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }).into(),
tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() })
.into(),
],
}
crate::TokenTree::Leaf(leaf) => match leaf {
crate::Leaf::Ident(ident) => tt::Leaf::from(tt::Ident {
text: ident.text.clone(),
id: tt::TokenId::unspecified(),
})
.into(),
crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(),
crate::Leaf::Var(v) => {
if v.text == "crate" {
// FIXME: Properly handle $crate token
tt::Leaf::from(tt::Ident {
text: "$crate".into(),
id: tt::TokenId::unspecified(),
})
.into()
} else if !ctx.bindings.contains(&v.text) {
// Note that it is possible to have a `$var` inside a macro which is not bound.
// For example:
// ```
// macro_rules! foo {
// ($a:ident, $b:ident, $c:tt) => {
// macro_rules! bar {
// ($bi:ident) => {
// fn $bi() -> u8 {$c}
// }
// }
// }
// ```
// We just treat it a normal tokens
tt::Subtree {
delimiter: tt::Delimiter::None,
token_trees: vec![
tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone })
.into(),
tt::Leaf::from(tt::Ident {
text: v.text.clone(),
id: tt::TokenId::unspecified(),
})
.into(),
],
.into();
Fragment::Tokens(tt)
} else {
let fragment = ctx.bindings.get(&v, &ctx.nesting)?.clone();
ctx.var_expanded = true;
fragment
};
Ok(res)
}
fn expand_repeat(
ctx: &mut ExpandCtx,
template: &tt::Subtree,
kind: RepeatKind,
separator: Option<Separator>,
) -> Result<Fragment, ExpandError> {
let mut buf: Vec<tt::TokenTree> = Vec::new();
ctx.nesting.push(0);
// Dirty hack to make macro-expansion terminate.
// This should be replaced by a propper macro-by-example implementation
let mut limit = 65536;
let mut has_seps = 0;
let mut counter = 0;
// We store the old var expanded value, and restore it later
// It is because before this `$repeat`,
// it is possible some variables already expanad in the same subtree
//
// `some_var_expanded` keep check if the deeper subtree has expanded variables
let mut some_var_expanded = false;
let old_var_expanded = ctx.var_expanded;
ctx.var_expanded = false;
while let Ok(mut t) = expand_subtree(ctx, template) {
t.delimiter = tt::Delimiter::None;
// if no var expanded in the child, we count it as a fail
if !ctx.var_expanded {
break;
}
// Reset `ctx.var_expandeded` to see if there is other expanded variable
// in the next matching
some_var_expanded = true;
ctx.var_expanded = false;
counter += 1;
limit -= 1;
if limit == 0 {
log::warn!(
"expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
template,
ctx
);
break;
}
let idx = ctx.nesting.pop().unwrap();
ctx.nesting.push(idx + 1);
push_subtree(&mut buf, t);
if let Some(ref sep) = separator {
match sep {
Separator::Ident(ident) => {
has_seps = 1;
buf.push(tt::Leaf::from(ident.clone()).into());
}
Separator::Literal(lit) => {
has_seps = 1;
buf.push(tt::Leaf::from(lit.clone()).into());
}
Separator::Puncts(puncts) => {
has_seps = puncts.len();
for punct in puncts {
buf.push(tt::Leaf::from(*punct).into());
}
.into()
} else {
let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone();
ctx.var_expanded = true;
return Ok(fragment);
}
}
crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
},
};
Ok(Fragment::Tokens(res))
}
if RepeatKind::ZeroOrOne == kind {
break;
}
}
// Restore the `var_expanded` by combining old one and the new one
ctx.var_expanded = some_var_expanded || old_var_expanded;
ctx.nesting.pop().unwrap();
for _ in 0..has_seps {
buf.pop();
}
if RepeatKind::OneOrMore == kind && counter == 0 {
return Err(ExpandError::UnexpectedToken);
}
// Check if it is a single token subtree without any delimiter
// e.g {Delimiter:None> ['>'] /Delimiter:None>}
let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into();
Ok(Fragment::Tokens(tt))
}
fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {

187
crates/ra_mbe/src/mbe_parser.rs
View file

@ -1,187 +0,0 @@
use crate::tt_cursor::TtCursor;
/// This module parses a raw `tt::TokenStream` into macro-by-example token
/// stream. This is a *mostly* identify function, expect for handling of
/// `$var:tt_kind` and `$(repeat),*` constructs.
use crate::ParseError;
pub(crate) fn parse(tt: &tt::Subtree) -> Result<crate::MacroRules, ParseError> {
let mut parser = TtCursor::new(tt);
let mut rules = Vec::new();
while !parser.is_eof() {
rules.push(parse_rule(&mut parser)?);
if let Err(e) = parser.expect_char(';') {
if !parser.is_eof() {
return Err(e);
}
break;
}
}
Ok(crate::MacroRules { rules })
}
fn parse_rule(p: &mut TtCursor) -> Result<crate::Rule, ParseError> {
let lhs = parse_subtree(p.eat_subtree()?, false)?;
p.expect_char('=')?;
p.expect_char('>')?;
let mut rhs = parse_subtree(p.eat_subtree()?, true)?;
rhs.delimiter = crate::Delimiter::None;
Ok(crate::Rule { lhs, rhs })
}
fn is_boolean_literal(lit: Option<&tt::TokenTree>) -> bool {
if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) = lit {
if lit.text == "true" || lit.text == "false" {
return true;
}
}
false
}
fn parse_subtree(tt: &tt::Subtree, transcriber: bool) -> Result<crate::Subtree, ParseError> {
let mut token_trees = Vec::new();
let mut p = TtCursor::new(tt);
while let Some(tt) = p.eat() {
let child: crate::TokenTree = match tt {
tt::TokenTree::Leaf(leaf) => match leaf {
tt::Leaf::Punct(tt::Punct { char: '$', spacing }) => {
// mbe var can be an ident or keyword, including `true` and `false`
if p.at_ident().is_some() || is_boolean_literal(p.current()) {
crate::Leaf::from(parse_var(&mut p, transcriber)?).into()
} else if let Some(tt::TokenTree::Subtree(_)) = p.current() {
parse_repeat(&mut p, transcriber)?.into()
} else {
// Treat it as normal punct
crate::Leaf::from(tt::Punct { char: '$', spacing: *spacing }).into()
}
}
tt::Leaf::Punct(punct) => crate::Leaf::from(*punct).into(),
tt::Leaf::Ident(tt::Ident { text, .. }) => {
crate::Leaf::from(crate::Ident { text: text.clone() }).into()
}
tt::Leaf::Literal(tt::Literal { text }) => {
crate::Leaf::from(crate::Literal { text: text.clone() }).into()
}
},
tt::TokenTree::Subtree(subtree) => parse_subtree(&subtree, transcriber)?.into(),
};
token_trees.push(child);
}
Ok(crate::Subtree { token_trees, delimiter: tt.delimiter })
}
fn parse_var(p: &mut TtCursor, transcriber: bool) -> Result<crate::Var, ParseError> {
let text = {
if is_boolean_literal(p.current()) {
let lit = p.eat_literal().unwrap();
lit.text.clone()
} else {
let ident = p.eat_ident().unwrap();
ident.text.clone()
}
};
let kind = if !transcriber && p.at_char(':') {
p.bump();
if let Some(ident) = p.eat_ident() {
Some(ident.text.clone())
} else {
p.rev_bump();
None
}
} else {
None
};
Ok(crate::Var { text, kind })
}
fn mk_repeat(
rep: char,
subtree: crate::Subtree,
separator: Option<crate::Separator>,
) -> Result<crate::Repeat, ParseError> {
let kind = match rep {
'*' => crate::RepeatKind::ZeroOrMore,
'+' => crate::RepeatKind::OneOrMore,
'?' => crate::RepeatKind::ZeroOrOne,
_ => return Err(ParseError::Expected(String::from("repeat"))),
};
Ok(crate::Repeat { subtree, kind, separator })
}
fn parse_repeat(p: &mut TtCursor, transcriber: bool) -> Result<crate::Repeat, ParseError> {
let subtree = p.eat_subtree()?;
let mut subtree = parse_subtree(subtree, transcriber)?;
subtree.delimiter = crate::Delimiter::None;
if let Some(rep) = p.at_punct() {
match rep.char {
'*' | '+' | '?' => {
p.bump();
return mk_repeat(rep.char, subtree, None);
}
_ => {}
}
}
let sep = p.eat_seperator().ok_or_else(|| ParseError::Expected(String::from("separator")))?;
let rep = p.eat_punct().ok_or_else(|| ParseError::Expected(String::from("repeat")))?;
mk_repeat(rep.char, subtree, Some(sep))
}
#[cfg(test)]
mod tests {
use ra_syntax::{ast, AstNode};
use super::*;
use crate::ast_to_token_tree;
#[test]
fn test_invalid_parse() {
expect_err("invalid", "subtree");
is_valid("($i:ident) => ()");
is_valid("($($i:ident)*) => ($_)");
is_valid("($($true:ident)*) => ($true)");
is_valid("($($false:ident)*) => ($false)");
expect_err("$i:ident => ()", "subtree");
expect_err("($i:ident) ()", "`=`");
expect_err("($($i:ident)_) => ()", "repeat");
}
fn expect_err(macro_body: &str, expected: &str) {
assert_eq!(
create_rules(&format_macro(macro_body)),
Err(ParseError::Expected(String::from(expected)))
);
}
fn is_valid(macro_body: &str) {
assert!(create_rules(&format_macro(macro_body)).is_ok());
}
fn format_macro(macro_body: &str) -> String {
format!(
"
macro_rules! foo {{
{}
}}
",
macro_body
)
}
fn create_rules(macro_definition: &str) -> Result<crate::MacroRules, ParseError> {
let source_file = ast::SourceFile::parse(macro_definition).ok().unwrap();
let macro_definition =
source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
let (definition_tt, _) =
ast_to_token_tree(&macro_definition.token_tree().unwrap()).unwrap();
parse(&definition_tt)
}
}

187
crates/ra_mbe/src/parser.rs Normal file
View file

@ -0,0 +1,187 @@
//! Parser recognizes special macro syntax, `$var` and `$(repeat)*`, in token
//! trees.
use ra_syntax::SmolStr;
use smallvec::SmallVec;
use crate::{tt_iter::TtIter, ExpandError};
#[derive(Debug)]
pub(crate) enum Op<'a> {
Var { name: &'a SmolStr, kind: Option<&'a SmolStr> },
Repeat { subtree: &'a tt::Subtree, kind: RepeatKind, separator: Option<Separator> },
TokenTree(&'a tt::TokenTree),
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum RepeatKind {
ZeroOrMore,
OneOrMore,
ZeroOrOne,
}
#[derive(Clone, Debug, Eq)]
pub(crate) enum Separator {
Literal(tt::Literal),
Ident(tt::Ident),
Puncts(SmallVec<[tt::Punct; 3]>),
}
// Note that when we compare a Separator, we just care about its textual value.
impl PartialEq for Separator {
fn eq(&self, other: &Separator) -> bool {
use Separator::*;
match (self, other) {
(Ident(ref a), Ident(ref b)) => a.text == b.text,
(Literal(ref a), Literal(ref b)) => a.text == b.text,
(Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => {
let a_iter = a.iter().map(|a| a.char);
let b_iter = b.iter().map(|b| b.char);
a_iter.eq(b_iter)
}
_ => false,
}
}
}
pub(crate) fn parse_template<'a>(
template: &'a tt::Subtree,
) -> impl Iterator<Item = Result<Op<'a>, ExpandError>> {
parse_inner(template, Mode::Template)
}
pub(crate) fn parse_pattern<'a>(
pattern: &'a tt::Subtree,
) -> impl Iterator<Item = Result<Op<'a>, ExpandError>> {
parse_inner(pattern, Mode::Pattern)
}
#[derive(Clone, Copy)]
enum Mode {
Pattern,
Template,
}
fn parse_inner<'a>(
src: &'a tt::Subtree,
mode: Mode,
) -> impl Iterator<Item = Result<Op<'a>, ExpandError>> {
let mut src = TtIter::new(src);
std::iter::from_fn(move || {
let first = src.next()?;
Some(next_op(first, &mut src, mode))
})
}
macro_rules! err {
($($tt:tt)*) => {
ExpandError::UnexpectedToken
};
}
macro_rules! bail {
($($tt:tt)*) => {
return Err(err!($($tt)*))
};
}
fn next_op<'a>(
first: &'a tt::TokenTree,
src: &mut TtIter<'a>,
mode: Mode,
) -> Result<Op<'a>, ExpandError> {
let res = match first {
tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '$', .. })) => {
let second = src.next().ok_or_else(|| err!("bad var 1"))?;
match second {
tt::TokenTree::Subtree(subtree) => {
let (separator, kind) = parse_repeat(src)?;
Op::Repeat { subtree, separator, kind }
}
tt::TokenTree::Leaf(leaf) => match leaf {
tt::Leaf::Punct(..) => Err(ExpandError::UnexpectedToken)?,
tt::Leaf::Ident(ident) => {
let name = &ident.text;
let kind = eat_fragment_kind(src, mode)?;
Op::Var { name, kind }
}
tt::Leaf::Literal(lit) => {
if is_boolean_literal(lit) {
let name = &lit.text;
let kind = eat_fragment_kind(src, mode)?;
Op::Var { name, kind }
} else {
bail!("bad var 2");
}
}
},
}
}
tt => Op::TokenTree(tt),
};
Ok(res)
}
fn eat_fragment_kind<'a>(
src: &mut TtIter<'a>,
mode: Mode,
) -> Result<Option<&'a SmolStr>, ExpandError> {
if let Mode::Pattern = mode {
src.expect_char(':').map_err(|()| err!("bad fragment specifier 1"))?;
let ident = src.expect_ident().map_err(|()| err!("bad fragment specifier 1"))?;
return Ok(Some(&ident.text));
};
Ok(None)
}
fn is_boolean_literal(lit: &tt::Literal) -> bool {
match lit.text.as_str() {
"true" | "false" => true,
_ => false,
}
}
///TOOD: impl for slice iter
fn parse_repeat(src: &mut TtIter) -> Result<(Option<Separator>, RepeatKind), ExpandError> {
let mut separator = Separator::Puncts(SmallVec::new());
for tt in src {
let tt = match tt {
tt::TokenTree::Leaf(leaf) => leaf,
tt::TokenTree::Subtree(_) => Err(ExpandError::InvalidRepeat)?,
};
let has_sep = match &separator {
Separator::Puncts(puncts) => puncts.len() != 0,
_ => true,
};
match tt {
tt::Leaf::Ident(_) | tt::Leaf::Literal(_) if has_sep => {
Err(ExpandError::InvalidRepeat)?
}
tt::Leaf::Ident(ident) => separator = Separator::Ident(ident.clone()),
tt::Leaf::Literal(lit) => separator = Separator::Literal(lit.clone()),
tt::Leaf::Punct(punct) => {
let repeat_kind = match punct.char {
'*' => RepeatKind::ZeroOrMore,
'+' => RepeatKind::OneOrMore,
'?' => RepeatKind::ZeroOrOne,
_ => {
match &mut separator {
Separator::Puncts(puncts) => {
if puncts.len() == 3 {
Err(ExpandError::InvalidRepeat)?
}
puncts.push(punct.clone())
}
_ => Err(ExpandError::InvalidRepeat)?,
}
continue;
}
};
let separator = if has_sep { Some(separator) } else { None };
return Ok((separator, repeat_kind));
}
}
}
Err(ExpandError::InvalidRepeat)
}

91
crates/ra_mbe/src/subtree_parser.rs
View file

@ -1,91 +0,0 @@
use crate::subtree_source::SubtreeTokenSource;
use ra_parser::{FragmentKind, TokenSource, TreeSink};
use ra_syntax::SyntaxKind;
use tt::buffer::{Cursor, TokenBuffer};
struct OffsetTokenSink<'a> {
cursor: Cursor<'a>,
error: bool,
}
impl<'a> OffsetTokenSink<'a> {
pub fn collect(&self, begin: Cursor<'a>) -> Vec<&'a tt::TokenTree> {
if !self.cursor.is_root() {
return vec![];
}
let mut curr = begin;
let mut res = vec![];
while self.cursor != curr {
if let Some(token) = curr.token_tree() {
res.push(token);
}
curr = curr.bump();
}
res
}
}
impl<'a> TreeSink for OffsetTokenSink<'a> {
fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) {
for _ in 0..n_tokens {
self.cursor = self.cursor.bump_subtree();
}
}
fn start_node(&mut self, _kind: SyntaxKind) {}
fn finish_node(&mut self) {}
fn error(&mut self, _error: ra_parser::ParseError) {
self.error = true;
}
}
pub(crate) struct Parser<'a> {
subtree: &'a tt::Subtree,
cur_pos: &'a mut usize,
}
impl<'a> Parser<'a> {
pub fn new(cur_pos: &'a mut usize, subtree: &'a tt::Subtree) -> Parser<'a> {
Parser { cur_pos, subtree }
}
pub fn parse_fragment(self, fragment_kind: FragmentKind) -> Option<tt::TokenTree> {
self.parse(|token_source, tree_skink| {
ra_parser::parse_fragment(token_source, tree_skink, fragment_kind)
})
}
fn parse<F>(self, f: F) -> Option<tt::TokenTree>
where
F: FnOnce(&mut dyn TokenSource, &mut dyn TreeSink),
{
let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]);
let mut src = SubtreeTokenSource::new(&buffer);
let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
f(&mut src, &mut sink);
let r = self.finish(buffer.begin(), &mut sink);
if sink.error {
return None;
}
r
}
fn finish(self, begin: Cursor, sink: &mut OffsetTokenSink) -> Option<tt::TokenTree> {
let res = sink.collect(begin);
*self.cur_pos += res.len();
match res.len() {
0 => None,
1 => Some(res[0].clone()),
_ => Some(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::None,
token_trees: res.into_iter().cloned().collect(),
})),
}
}
}

48
crates/ra_mbe/src/tests.rs
View file

@ -3,6 +3,54 @@ use test_utils::assert_eq_text;
use super::*;
mod rule_parsing {
use ra_syntax::{ast, AstNode};
use super::*;
use crate::ast_to_token_tree;
#[test]
fn test_valid_arms() {
fn check(macro_body: &str) {
let m = parse_macro_arm(macro_body);
m.unwrap();
}
check("($i:ident) => ()");
check("($($i:ident)*) => ($_)");
check("($($true:ident)*) => ($true)");
check("($($false:ident)*) => ($false)");
}
#[test]
fn test_invalid_arms() {
fn check(macro_body: &str, err: &str) {
let m = parse_macro_arm(macro_body);
assert_eq!(m, Err(ParseError::Expected(String::from(err))));
}
check("invalid", "expected subtree");
check("$i:ident => ()", "expected subtree");
check("($i:ident) ()", "expected `=`");
check("($($i:ident)_) => ()", "invalid repeat");
check("($i) => ($i)", "invalid macro definition");
check("($i:) => ($i)", "invalid macro definition");
}
fn parse_macro_arm(arm_definition: &str) -> Result<crate::MacroRules, ParseError> {
let macro_definition = format!(" macro_rules! m {{ {} }} ", arm_definition);
let source_file = ast::SourceFile::parse(&macro_definition).ok().unwrap();
let macro_definition =
source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
let (definition_tt, _) =
ast_to_token_tree(&macro_definition.token_tree().unwrap()).unwrap();
crate::MacroRules::parse(&definition_tt)
}
}
// Good first issue (although a slightly challenging one):
//
// * Pick a random test from here

281
crates/ra_mbe/src/tt_cursor.rs
View file

@ -1,281 +0,0 @@
use crate::{subtree_parser::Parser, ParseError};
use ra_parser::FragmentKind;
use smallvec::{smallvec, SmallVec};
#[derive(Debug, Clone)]
pub(crate) struct TtCursor<'a> {
subtree: &'a tt::Subtree,
pos: usize,
}
pub(crate) struct TtCursorMemento {
pos: usize,
}
impl<'a> TtCursor<'a> {
pub(crate) fn new(subtree: &'a tt::Subtree) -> TtCursor<'a> {
TtCursor { subtree, pos: 0 }
}
pub(crate) fn is_eof(&self) -> bool {
self.pos == self.subtree.token_trees.len()
}
pub(crate) fn current(&self) -> Option<&'a tt::TokenTree> {
self.subtree.token_trees.get(self.pos)
}
pub(crate) fn at_punct(&self) -> Option<&'a tt::Punct> {
match self.current() {
Some(tt::TokenTree::Leaf(tt::Leaf::Punct(it))) => Some(it),
_ => None,
}
}
pub(crate) fn at_char(&self, char: char) -> bool {
match self.at_punct() {
Some(tt::Punct { char: c, .. }) if *c == char => true,
_ => false,
}
}
pub(crate) fn at_ident(&mut self) -> Option<&'a tt::Ident> {
match self.current() {
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(i))) => Some(i),
_ => None,
}
}
pub(crate) fn at_literal(&mut self) -> Option<&'a tt::Literal> {
match self.current() {
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(i))) => Some(i),
_ => None,
}
}
pub(crate) fn bump(&mut self) {
self.pos += 1;
}
pub(crate) fn rev_bump(&mut self) {
self.pos -= 1;
}
pub(crate) fn eat(&mut self) -> Option<&'a tt::TokenTree> {
self.current().map(|it| {
self.bump();
it
})
}
pub(crate) fn eat_subtree(&mut self) -> Result<&'a tt::Subtree, ParseError> {
match self.current() {
Some(tt::TokenTree::Subtree(sub)) => {
self.bump();
Ok(sub)
}
_ => Err(ParseError::Expected(String::from("subtree"))),
}
}
pub(crate) fn eat_punct(&mut self) -> Option<&'a tt::Punct> {
self.at_punct().map(|it| {
self.bump();
it
})
}
pub(crate) fn eat_ident(&mut self) -> Option<&'a tt::Ident> {
self.at_ident().map(|i| {
self.bump();
i
})
}
pub(crate) fn eat_literal(&mut self) -> Option<&'a tt::Literal> {
self.at_literal().map(|i| {
self.bump();
i
})
}
pub(crate) fn eat_fragment(&mut self, fragment_kind: FragmentKind) -> Option<tt::TokenTree> {
let parser = Parser::new(&mut self.pos, self.subtree);
parser.parse_fragment(fragment_kind)
}
pub(crate) fn eat_lifetime(&mut self) -> Option<tt::TokenTree> {
// check if it start from "`"
if let Some(ident) = self.at_ident() {
if ident.text.chars().next()? != '\'' {
return None;
}
}
self.eat_ident().cloned().map(|ident| tt::Leaf::from(ident).into())
}
pub(crate) fn try_eat_vis(&mut self) -> Option<tt::TokenTree> {
// `vis` matcher is optional
let old_pos = self.pos;
let parser = Parser::new(&mut self.pos, self.subtree);
let res = parser.parse_fragment(FragmentKind::Visibility);
if res.is_none() {
self.pos = old_pos;
}
res
}
pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> {
if self.at_char(char) {
self.bump();
Ok(())
} else {
Err(ParseError::Expected(format!("`{}`", char)))
}
}
fn eat_punct3(&mut self, p: tt::Punct) -> Option<SmallVec<[tt::Punct; 3]>> {
let sec = *self.eat_punct()?;
let third = *self.eat_punct()?;
Some(smallvec![p, sec, third])
}
fn eat_punct2(&mut self, p: tt::Punct) -> Option<SmallVec<[tt::Punct; 3]>> {
let sec = *self.eat_punct()?;
Some(smallvec![p, sec])
}
fn eat_multi_char_punct<'b, I>(
&mut self,
p: tt::Punct,
iter: &mut TokenPeek<'b, I>,
) -> Option<SmallVec<[tt::Punct; 3]>>
where
I: Iterator<Item = &'b tt::TokenTree>,
{
if let Some((m, _)) = iter.current_punct3(p) {
if let r @ Some(_) = match m {
('<', '<', '=') | ('>', '>', '=') | ('.', '.', '.') | ('.', '.', '=') => {
self.eat_punct3(p)
}
_ => None,
} {
return r;
}
}
if let Some((m, _)) = iter.current_punct2(p) {
if let r @ Some(_) = match m {
('<', '=')
| ('>', '=')
| ('+', '=')
| ('-', '=')
| ('|', '=')
| ('&', '=')
| ('^', '=')
| ('/', '=')
| ('*', '=')
| ('%', '=')
| ('&', '&')
| ('|', '|')
| ('<', '<')
| ('>', '>')
| ('-', '>')
| ('!', '=')
| ('=', '>')
| ('=', '=')
| ('.', '.')
| (':', ':') => self.eat_punct2(p),
_ => None,
} {
return r;
}
}
None
}
pub(crate) fn eat_seperator(&mut self) -> Option<crate::Separator> {
match self.eat()? {
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
Some(crate::Separator::Literal(lit.clone()))
}
tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
Some(crate::Separator::Ident(ident.clone()))
}
tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => {
match punct.char {
'*' | '+' | '?' => return None,
_ => {}
};
// FIXME: The parser is only handle some compositeable punct,
// But at this phase, some punct still is jointed.
// So we by pass that check here.
let mut peekable = TokenPeek::new(self.subtree.token_trees[self.pos..].iter());
let puncts = self.eat_multi_char_punct(*punct, &mut peekable);
let puncts = puncts.unwrap_or_else(|| smallvec![*punct]);
Some(crate::Separator::Puncts(puncts))
}
_ => None,
}
}
#[must_use]
pub(crate) fn save(&self) -> TtCursorMemento {
TtCursorMemento { pos: self.pos }
}
pub(crate) fn rollback(&mut self, memento: TtCursorMemento) {
self.pos = memento.pos;
}
}
pub(crate) struct TokenPeek<'a, I>
where
I: Iterator<Item = &'a tt::TokenTree>,
{
iter: itertools::MultiPeek<I>,
}
// helper function
fn to_punct(tt: &tt::TokenTree) -> Option<&tt::Punct> {
if let tt::TokenTree::Leaf(tt::Leaf::Punct(pp)) = tt {
return Some(pp);
}
None
}
impl<'a, I> TokenPeek<'a, I>
where
I: Iterator<Item = &'a tt::TokenTree>,
{
pub fn new(iter: I) -> Self {
TokenPeek { iter: itertools::multipeek(iter) }
}
pub fn current_punct2(&mut self, p: tt::Punct) -> Option<((char, char), bool)> {
if p.spacing != tt::Spacing::Joint {
return None;
}
self.iter.reset_peek();
let p1 = to_punct(self.iter.peek()?)?;
Some(((p.char, p1.char), p1.spacing == tt::Spacing::Joint))
}
pub fn current_punct3(&mut self, p: tt::Punct) -> Option<((char, char, char), bool)> {
self.current_punct2(p).and_then(|((p0, p1), last_joint)| {
if !last_joint {
None
} else {
let p2 = to_punct(*self.iter.peek()?)?;
Some(((p0, p1, p2.char), p2.spacing == tt::Spacing::Joint))
}
})
}
}

67
crates/ra_mbe/src/tt_iter.rs Normal file
View file

@ -0,0 +1,67 @@
#[derive(Debug, Clone)]
pub(crate) struct TtIter<'a> {
pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>,
}
impl<'a> TtIter<'a> {
pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> {
TtIter { inner: subtree.token_trees.iter() }
}
pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ()> {
match self.next() {
Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: c, .. }))) if *c == char => {
Ok(())
}
_ => Err(()),
}
}
pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> {
match self.next() {
Some(tt::TokenTree::Subtree(it)) => Ok(it),
_ => Err(()),
}
}
pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> {
match self.next() {
Some(tt::TokenTree::Leaf(it)) => Ok(it),
_ => Err(()),
}
}
pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> {
match self.expect_leaf()? {
tt::Leaf::Ident(it) => Ok(it),
_ => Err(()),
}
}
pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Literal, ()> {
match self.expect_leaf()? {
tt::Leaf::Literal(it) => Ok(it),
_ => Err(()),
}
}
pub(crate) fn expect_punct(&mut self) -> Result<&'a tt::Punct, ()> {
match self.expect_leaf()? {
tt::Leaf::Punct(it) => Ok(it),
_ => Err(()),
}
}
}
impl<'a> Iterator for TtIter<'a> {
type Item = &'a tt::TokenTree;
fn next(&mut self) -> Option<Self::Item> {
self.inner.next()
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
}
impl<'a> std::iter::ExactSizeIterator for TtIter<'a> {}