Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2025-01-13 13:48:50 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add token id to delims

Browse source
This commit is contained in:
Edwin Cheng 2019-12-13 01:41:44 +08:00
parent aceb9d7fb0
commit 59295854f8
6 changed files with 134 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
crates/ra_hir_expand/src/quote.rs
View file

@ -16,7 +16,10 @@ macro_rules! __quote {
{ {
let children = $crate::__quote!($($tt)*); let children = $crate::__quote!($($tt)*);
let subtree = tt::Subtree { let subtree = tt::Subtree {
delimiter: Some(tt::Delimiter::$delim), delimiter: Some(tt::Delimiter {
kind: tt::DelimiterKind::$delim,
id: tt::TokenId::unspecified(),
}),
token_trees: $crate::quote::IntoTt::to_tokens(children), token_trees: $crate::quote::IntoTt::to_tokens(children),
}; };
subtree subtree
@ -257,8 +260,13 @@ mod tests {
let fields = let fields =
fields.iter().map(|it| quote!(#it: self.#it.clone(), ).token_trees.clone()).flatten(); fields.iter().map(|it| quote!(#it: self.#it.clone(), ).token_trees.clone()).flatten();
let list = let list = tt::Subtree {
tt::Subtree { delimiter: Some(tt::Delimiter::Brace), token_trees: fields.collect() }; delimiter: Some(tt::Delimiter {
kind: tt::DelimiterKind::Brace,
id: tt::TokenId::unspecified(),
}),
token_trees: fields.collect(),
};
let quoted = quote! { let quoted = quote! {
impl Clone for #struct_name { impl Clone for #struct_name {

2
crates/ra_mbe/src/mbe_expander/matcher.rs
View file

@ -106,7 +106,7 @@ fn match_subtree(
} }
Op::TokenTree(tt::TokenTree::Subtree(lhs)) => { Op::TokenTree(tt::TokenTree::Subtree(lhs)) => {
let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?; let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?;
if lhs.delimiter != rhs.delimiter { if lhs.delimiter.map(|it| it.kind) != rhs.delimiter.map(|it| it.kind) {
bail!("mismatched delimiter") bail!("mismatched delimiter")
} }
let mut src = TtIter::new(rhs); let mut src = TtIter::new(rhs);

8
crates/ra_mbe/src/subtree_source.rs
View file

@ -115,10 +115,10 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> {
} }
fn convert_delim(d: Option<tt::Delimiter>, closing: bool) -> TtToken { fn convert_delim(d: Option<tt::Delimiter>, closing: bool) -> TtToken {
let (kinds, texts) = match d { let (kinds, texts) = match d.map(|it| it.kind) {
Some(tt::Delimiter::Parenthesis) => ([T!['('], T![')']], "()"), Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"),
Some(tt::Delimiter::Brace) => ([T!['{'], T!['}']], "{}"), Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"),
Some(tt::Delimiter::Bracket) => ([T!['['], T![']']], "[]"), Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"),
None => ([L_DOLLAR, R_DOLLAR], ""), None => ([L_DOLLAR, R_DOLLAR], ""),
}; };

144
crates/ra_mbe/src/syntax_bridge.rs
View file

@ -5,6 +5,7 @@ use ra_syntax::{
ast, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode, ast, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode,
SyntaxTreeBuilder, TextRange, TextUnit, T, SyntaxTreeBuilder, TextRange, TextUnit, T,
}; };
use rustc_hash::FxHashMap;
use std::iter::successors; use std::iter::successors;
use tt::buffer::{Cursor, TokenBuffer}; use tt::buffer::{Cursor, TokenBuffer};
@ -83,6 +84,15 @@ impl TokenMap {
fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) { fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) {
self.entries.push((token_id, relative_range)); self.entries.push((token_id, relative_range));
} }
fn insert_delim(
&mut self,
_token_id: tt::TokenId,
_open_relative_range: TextRange,
_close_relative_range: TextRange,
) {
// FIXME: Add entries for delimiter
}
} }
/// Returns the textual content of a doc comment block as a quoted string /// Returns the textual content of a doc comment block as a quoted string
@ -121,7 +131,10 @@ fn convert_doc_comment(token: &ra_syntax::SyntaxToken) -> Option<Vec<tt::TokenTr
token_trees.push(mk_punct('!')); token_trees.push(mk_punct('!'));
} }
token_trees.push(tt::TokenTree::from(tt::Subtree { token_trees.push(tt::TokenTree::from(tt::Subtree {
delimiter: Some(tt::Delimiter::Bracket), delimiter: Some(tt::Delimiter {
kind: tt::DelimiterKind::Bracket,
id: tt::TokenId::unspecified(),
}),
token_trees: meta_tkns, token_trees: meta_tkns,
})); }));
@ -190,12 +203,16 @@ impl Convertor {
.last() .last()
.unwrap(); .unwrap();
let (delimiter, skip_first) = match (first_child.kind(), last_child.kind()) { let (delimiter_kind, skip_first) = match (first_child.kind(), last_child.kind()) {
(T!['('], T![')']) => (Some(tt::Delimiter::Parenthesis), true), (T!['('], T![')']) => (Some(tt::DelimiterKind::Parenthesis), true),
(T!['{'], T!['}']) => (Some(tt::Delimiter::Brace), true), (T!['{'], T!['}']) => (Some(tt::DelimiterKind::Brace), true),
(T!['['], T![']']) => (Some(tt::Delimiter::Bracket), true), (T!['['], T![']']) => (Some(tt::DelimiterKind::Bracket), true),
_ => (None, false), _ => (None, false),
}; };
let delimiter = delimiter_kind.map(|kind| tt::Delimiter {
kind,
id: self.alloc_delim(first_child.text_range(), last_child.text_range()),
});
let mut token_trees = Vec::new(); let mut token_trees = Vec::new();
let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable(); let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable();
@ -232,25 +249,31 @@ impl Convertor {
.push(tt::Leaf::from(tt::Punct { char, spacing, id }).into()); .push(tt::Leaf::from(tt::Punct { char, spacing, id }).into());
} }
} else { } else {
let child: tt::TokenTree = if token.kind() == T![true] let child: tt::Leaf = match token.kind() {
|| token.kind() == T![false] T![true] | T![false] => {
{ let id = self.alloc(token.text_range());
let id = self.alloc(token.text_range()); let text = token.text().clone();
tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into() tt::Literal { text, id }.into()
} else if token.kind().is_keyword() }
|| token.kind() == IDENT IDENT | LIFETIME => {
|| token.kind() == LIFETIME let id = self.alloc(token.text_range());
{ let text = token.text().clone();
let id = self.alloc(token.text_range()); tt::Ident { text, id }.into()
let text = token.text().clone(); }
tt::Leaf::from(tt::Ident { text, id }).into() k if k.is_keyword() => {
} else if token.kind().is_literal() { let id = self.alloc(token.text_range());
let id = self.alloc(token.text_range()); let text = token.text().clone();
tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into() tt::Ident { text, id }.into()
} else { }
return None; k if k.is_literal() => {
let id = self.alloc(token.text_range());
let text = token.text().clone();
tt::Literal { text, id }.into()
}
_ => return None,
}; };
token_trees.push(child);
token_trees.push(child.into());
} }
} }
NodeOrToken::Node(node) => { NodeOrToken::Node(node) => {
@ -275,11 +298,26 @@ impl Convertor {
self.map.insert(token_id, relative_range); self.map.insert(token_id, relative_range);
token_id token_id
} }
fn alloc_delim(
&mut self,
open_abs_range: TextRange,
close_abs_range: TextRange,
) -> tt::TokenId {
let open_relative_range = open_abs_range - self.global_offset;
let close_relative_range = close_abs_range - self.global_offset;
let token_id = tt::TokenId(self.next_id);
self.next_id += 1;
self.map.insert_delim(token_id, open_relative_range, close_relative_range);
token_id
}
} }
struct TtTreeSink<'a> { struct TtTreeSink<'a> {
buf: String, buf: String,
cursor: Cursor<'a>, cursor: Cursor<'a>,
open_delims: FxHashMap<tt::TokenId, TextUnit>,
text_pos: TextUnit, text_pos: TextUnit,
inner: SyntaxTreeBuilder, inner: SyntaxTreeBuilder,
token_map: TokenMap, token_map: TokenMap,
@ -294,6 +332,7 @@ impl<'a> TtTreeSink<'a> {
TtTreeSink { TtTreeSink {
buf: String::new(), buf: String::new(),
cursor, cursor,
open_delims: FxHashMap::default(),
text_pos: 0.into(), text_pos: 0.into(),
inner: SyntaxTreeBuilder::default(), inner: SyntaxTreeBuilder::default(),
roots: smallvec::SmallVec::new(), roots: smallvec::SmallVec::new(),
@ -307,10 +346,10 @@ impl<'a> TtTreeSink<'a> {
} }
fn delim_to_str(d: Option<tt::Delimiter>, closing: bool) -> SmolStr { fn delim_to_str(d: Option<tt::Delimiter>, closing: bool) -> SmolStr {
let texts = match d { let texts = match d.map(|it| it.kind) {
Some(tt::Delimiter::Parenthesis) => "()", Some(tt::DelimiterKind::Parenthesis) => "()",
Some(tt::Delimiter::Brace) => "{}", Some(tt::DelimiterKind::Brace) => "{}",
Some(tt::Delimiter::Bracket) => "[]", Some(tt::DelimiterKind::Bracket) => "[]",
None => return "".into(), None => return "".into(),
}; };
@ -331,34 +370,49 @@ impl<'a> TreeSink for TtTreeSink<'a> {
break; break;
} }
match self.cursor.token_tree() { let text: Option<SmolStr> = match self.cursor.token_tree() {
Some(tt::TokenTree::Leaf(leaf)) => { Some(tt::TokenTree::Leaf(leaf)) => {
// Mark the range if needed // Mark the range if needed
if let tt::Leaf::Ident(ident) = leaf { let id = match leaf {
if kind == IDENT { tt::Leaf::Ident(ident) => ident.id,
let range = tt::Leaf::Punct(punct) => punct.id,
TextRange::offset_len(self.text_pos, TextUnit::of_str(&ident.text)); tt::Leaf::Literal(lit) => lit.id,
self.token_map.insert(ident.id, range); };
} let text = SmolStr::new(format!("{}", leaf));
} let range = TextRange::offset_len(self.text_pos, TextUnit::of_str(&text));
self.token_map.insert(id, range);
self.cursor = self.cursor.bump(); self.cursor = self.cursor.bump();
self.buf += &format!("{}", leaf); Some(text)
} }
Some(tt::TokenTree::Subtree(subtree)) => { Some(tt::TokenTree::Subtree(subtree)) => {
self.cursor = self.cursor.subtree().unwrap(); self.cursor = self.cursor.subtree().unwrap();
self.buf += &delim_to_str(subtree.delimiter, false); if let Some(id) = subtree.delimiter.map(|it| it.id) {
} self.open_delims.insert(id, self.text_pos);
None => {
if let Some(parent) = self.cursor.end() {
self.cursor = self.cursor.bump();
self.buf += &delim_to_str(parent.delimiter, true);
} }
Some(delim_to_str(subtree.delimiter, false))
} }
None => self.cursor.end().and_then(|parent| {
self.cursor = self.cursor.bump();
if let Some(id) = parent.delimiter.map(|it| it.id) {
if let Some(open_delim) = self.open_delims.get(&id) {
let open_range =
TextRange::offset_len(*open_delim, TextUnit::from_usize(1));
let close_range =
TextRange::offset_len(self.text_pos, TextUnit::from_usize(1));
self.token_map.insert_delim(id, open_range, close_range);
}
}
Some(delim_to_str(parent.delimiter, true))
}),
}; };
if let Some(text) = text {
self.buf += &text;
self.text_pos += TextUnit::of_str(&text);
}
} }
self.text_pos += TextUnit::of_str(&self.buf);
let text = SmolStr::new(self.buf.as_str()); let text = SmolStr::new(self.buf.as_str());
self.buf.clear(); self.buf.clear();
self.inner.token(kind, text); self.inner.token(kind, text);
@ -504,7 +558,7 @@ mod tests {
let token_tree = ast::TokenTree::cast(token_tree).unwrap(); let token_tree = ast::TokenTree::cast(token_tree).unwrap();
let tt = ast_to_token_tree(&token_tree).unwrap().0; let tt = ast_to_token_tree(&token_tree).unwrap().0;
assert_eq!(tt.delimiter, Some(tt::Delimiter::Brace)); assert_eq!(tt.delimiter.map(|it| it.kind), Some(tt::DelimiterKind::Brace));
} }
#[test] #[test]

14
crates/ra_mbe/src/tests.rs
View file

@ -77,13 +77,15 @@ macro_rules! foobar {
} }
assert_eq!(expansion.token_trees.len(), 3); assert_eq!(expansion.token_trees.len(), 3);
// ($e:ident) => { foo bar $e } // {($e:ident) => { foo bar $e }}
// 0123 45 6 7 89 // 012345 67 8 9 T 12
assert_eq!(get_id(&expansion.token_trees[0]), Some(6)); assert_eq!(get_id(&expansion.token_trees[0]), Some(9));
assert_eq!(get_id(&expansion.token_trees[1]), Some(7)); assert_eq!(get_id(&expansion.token_trees[1]), Some(10));
// So baz should be 10 // The input args of macro call include parentheses:
assert_eq!(get_id(&expansion.token_trees[2]), Some(10)); // (baz)
// So baz should be 12+1+1
assert_eq!(get_id(&expansion.token_trees[2]), Some(14));
} }
#[test] #[test]

16
crates/ra_tt/src/lib.rs
View file

@ -55,7 +55,13 @@ pub struct Subtree {
} }
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Delimiter { pub struct Delimiter {
pub id: TokenId,
pub kind: DelimiterKind,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum DelimiterKind {
Parenthesis, Parenthesis,
Brace, Brace,
Bracket, Bracket,
@ -97,10 +103,10 @@ impl fmt::Display for TokenTree {
impl fmt::Display for Subtree { impl fmt::Display for Subtree {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let (l, r) = match self.delimiter { let (l, r) = match self.delimiter.map(|it| it.kind) {
Some(Delimiter::Parenthesis) => ("(", ")"), Some(DelimiterKind::Parenthesis) => ("(", ")"),
Some(Delimiter::Brace) => ("{", "}"), Some(DelimiterKind::Brace) => ("{", "}"),
Some(Delimiter::Bracket) => ("[", "]"), Some(DelimiterKind::Bracket) => ("[", "]"),
None => ("", ""), None => ("", ""),
}; };
f.write_str(l)?; f.write_str(l)?;