Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-11-10 07:04:22 +00:00
Code Issues Projects Releases Packages Wiki Activity

"Fix" mbe to work with decomposed tokens

Browse source 
We regressed $i * 2 where $i = 1 + 1, need to fix that!
This commit is contained in:
Aleksey Kladov 2019-09-10 14:27:08 +03:00
parent 40170885e7
commit 1c5800dee8
6 changed files with 46 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -1051,6 +1051,7 @@ dependencies = [
"ra_tt 0.1.0",
"rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)",
"test_utils 0.1.0",
]
[[package]]

4
crates/ra_mbe/Cargo.toml
View file

@ -12,3 +12,7 @@ itertools = "0.8.0"
rustc-hash = "1.0.0"
smallvec = "0.6.9"
log = "0.4.5"
[dev-dependencies]
test_utils = { path = "../test_utils" }

50
crates/ra_mbe/src/mbe_expander.rs
View file

@ -339,21 +339,13 @@ fn expand_subtree(
template: &crate::Subtree,
ctx: &mut ExpandCtx,
) -> Result<tt::Subtree, ExpandError> {
let token_trees = template
.token_trees
.iter()
.map(|it| expand_tt(it, ctx))
.filter(|it| {
// Filter empty subtree
if let Ok(tt::TokenTree::Subtree(subtree)) = it {
subtree.delimiter != tt::Delimiter::None || !subtree.token_trees.is_empty()
} else {
true
}
})
.collect::<Result<Vec<_>, ExpandError>>()?;
let mut buf: Vec<tt::TokenTree> = Vec::new();
for tt in template.token_trees.iter() {
let tt = expand_tt(tt, ctx)?;
push_tt(&mut buf, tt);
}
Ok(tt::Subtree { token_trees, delimiter: template.delimiter })
Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf })
}
/// Reduce single token subtree to single token
@ -377,7 +369,7 @@ fn expand_tt(
let res: tt::TokenTree = match template {
crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(),
crate::TokenTree::Repeat(repeat) => {
let mut token_trees: Vec<tt::TokenTree> = Vec::new();
let mut buf: Vec<tt::TokenTree> = Vec::new();
ctx.nesting.push(0);
// Dirty hack to make macro-expansion terminate.
// This should be replaced by a propper macro-by-example implementation
@ -418,23 +410,23 @@ fn expand_tt(
let idx = ctx.nesting.pop().unwrap();
ctx.nesting.push(idx + 1);
token_trees.push(reduce_single_token(t));
push_subtree(&mut buf, t);
if let Some(ref sep) = repeat.separator {
match sep {
crate::Separator::Ident(ident) => {
has_seps = 1;
token_trees.push(tt::Leaf::from(ident.clone()).into());
buf.push(tt::Leaf::from(ident.clone()).into());
}
crate::Separator::Literal(lit) => {
has_seps = 1;
token_trees.push(tt::Leaf::from(lit.clone()).into());
buf.push(tt::Leaf::from(lit.clone()).into());
}
crate::Separator::Puncts(puncts) => {
has_seps = puncts.len();
for punct in puncts {
token_trees.push(tt::Leaf::from(*punct).into());
buf.push(tt::Leaf::from(*punct).into());
}
}
}
@ -450,16 +442,16 @@ fn expand_tt(
ctx.nesting.pop().unwrap();
for _ in 0..has_seps {
token_trees.pop();
buf.pop();
}
if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 {
return Err(ExpandError::UnexpectedToken);
}
// Check if it is a singel token subtree without any delimiter
// Check if it is a single token subtree without any delimiter
// e.g {Delimiter:None> ['>'] /Delimiter:None>}
reduce_single_token(tt::Subtree { token_trees, delimiter: tt::Delimiter::None })
reduce_single_token(tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf })
}
crate::TokenTree::Leaf(leaf) => match leaf {
crate::Leaf::Ident(ident) => {
@ -586,3 +578,17 @@ mod tests {
expand_rule(&rules.rules[0], &invocation_tt)
}
}
fn push_tt(buf: &mut Vec<tt::TokenTree>, tt: tt::TokenTree) {
match tt {
tt::TokenTree::Subtree(tt) => push_subtree(buf, tt),
_ => buf.push(tt),
}
}
fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
match tt.delimiter {
tt::Delimiter::None => buf.extend(tt.token_trees),
_ => buf.push(tt.into()),
}
}

9
crates/ra_mbe/src/syntax_bridge.rs
View file

@ -70,7 +70,14 @@ fn fragment_to_syntax_node(
tt: &tt::Subtree,
fragment_kind: FragmentKind,
) -> Result<Parse<SyntaxNode>, ExpandError> {
let tokens = [tt.clone().into()];
let tmp;
let tokens = match tt {
tt::Subtree { delimiter: tt::Delimiter::None, token_trees } => token_trees.as_slice(),
_ => {
tmp = [tt.clone().into()];
&tmp[..]
}
};
let buffer = TokenBuffer::new(&tokens);
let mut token_source = SubtreeTokenSource::new(&buffer);
let mut tree_sink = TtTreeSink::new(buffer.begin());

9
crates/ra_mbe/src/tests.rs
View file

@ -1,4 +1,5 @@
use ra_syntax::{ast, AstNode, NodeOrToken};
use test_utils::assert_eq_text;
use super::*;
@ -152,7 +153,6 @@ pub(crate) fn assert_expansion(
// wrap the given text to a macro call
let expected = text_to_tokentree(&expected);
let (expanded_tree, expected_tree) = match kind {
MacroKind::Items => {
let expanded_tree = token_tree_to_macro_items(&expanded).unwrap().tree();
@ -178,7 +178,7 @@ pub(crate) fn assert_expansion(
let expected_tree = expected_tree.replace("C_C__C", "$crate");
assert_eq!(
expanded_tree, expected_tree,
"left => {}\nright => {}",
"\nleft:\n{}\nright:\n{}",
expanded_tree, expected_tree,
);
@ -657,6 +657,7 @@ fn test_expr() {
}
#[test]
#[ignore]
fn test_expr_order() {
let rules = create_rules(
r#"
@ -668,8 +669,8 @@ fn test_expr_order() {
"#,
);
assert_eq!(
format!("{:#?}", expand_to_items(&rules, "foo! { 1 + 1 }").syntax()).trim(),
assert_eq_text!(
&format!("{:#?}", expand_to_items(&rules, "foo! { 1 + 1 }").syntax()),
r#"MACRO_ITEMS@[0; 15)
FN_DEF@[0; 15)
FN_KW@[0; 2) "fn"

57
crates/ra_parser/src/parser.rs
View file

@ -50,29 +50,6 @@ impl<'t> Parser<'t> {
self.steps.set(steps + 1);
self.token_source.lookahead_nth(n).kind
// // It is because the Dollar will appear between nth
// // Following code skips through it
// let mut non_dollars_count = 0;
// let mut i = 0;
// loop {
// let token = self.token_source.lookahead_nth(i);
// let mut kind = token.kind;
// if let Some((composited, step)) = self.is_composite(token, i) {
// kind = composited;
// i += step;
// } else {
// i += 1;
// }
// match kind {
// EOF => return EOF,
// SyntaxKind::L_DOLLAR | SyntaxKind::R_DOLLAR => {}
// _ if non_dollars_count == n => return kind,
// _ => non_dollars_count += 1,
// }
// }
}
/// Checks if the current token is `kind`.
@ -185,25 +162,6 @@ impl<'t> Parser<'t> {
assert!(self.eat(kind));
}
/// Advances the parser by one token unconditionally
/// Mainly use in `token_tree` parsing
#[allow(unused)]
fn bump_raw(&mut self) {
let mut kind = self.token_source.current().kind;
// Skip dollars, do_bump will eat these later
let mut i = 0;
while kind == SyntaxKind::L_DOLLAR || kind == SyntaxKind::R_DOLLAR {
kind = self.token_source.lookahead_nth(i).kind;
i += 1;
}
if kind == EOF {
return;
}
self.do_bump(kind, 1);
}
/// Advances the parser by one token with composite puncts handled
pub(crate) fn bump_any(&mut self) {
let kind = self.nth(0);
@ -277,21 +235,6 @@ impl<'t> Parser<'t> {
self.events.push(event)
}
#[allow(unused)]
fn eat_dollars(&mut self) {
loop {
match self.token_source.current().kind {
k @ SyntaxKind::L_DOLLAR | k @ SyntaxKind::R_DOLLAR => {
self.token_source.bump();
self.push_event(Event::Token { kind: k, n_raw_tokens: 1 });
}
_ => {
return;
}
}
}
}
pub(crate) fn eat_l_dollars(&mut self) -> usize {
let mut ate_count = 0;
loop {