Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2025-01-14 14:13:58 +00:00
Code Issues Projects Releases Packages Wiki Activity

Simplify SubtreeTokenSource

Browse source
This commit is contained in:
Edwin Cheng 2021-01-05 01:50:34 +08:00
parent af3d75ad2e
commit d387bfdc4a
1 changed files with 60 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

142
crates/mbe/src/subtree_source.rs
View file

@ -1,131 +1,104 @@
//! FIXME: write short doc here //! FIXME: write short doc here
use parser::{Token, TokenSource}; use parser::{Token, TokenSource};
use std::cell::{Cell, Ref, RefCell};
use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T}; use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
use tt::buffer::{Cursor, TokenBuffer}; use tt::buffer::TokenBuffer;
#[derive(Debug, Clone, Eq, PartialEq)] #[derive(Debug, Clone, Eq, PartialEq)]
struct TtToken { struct TtToken {
kind: SyntaxKind, tt: Token,
is_joint_to_next: bool,
text: SmolStr, text: SmolStr,
} }
pub(crate) struct SubtreeTokenSource<'a> { pub(crate) struct SubtreeTokenSource {
cached_cursor: Cell<Cursor<'a>>, cached: Vec<TtToken>,
cached: RefCell<Vec<Option<TtToken>>>,
curr: (Token, usize), curr: (Token, usize),
} }
impl<'a> SubtreeTokenSource<'a> { impl<'a> SubtreeTokenSource {
// Helper function used in test // Helper function used in test
#[cfg(test)] #[cfg(test)]
pub(crate) fn text(&self) -> SmolStr { pub(crate) fn text(&self) -> SmolStr {
match *self.get(self.curr.1) { match self.cached.get(self.curr.1) {
Some(ref tt) => tt.text.clone(), Some(ref tt) => tt.text.clone(),
_ => SmolStr::new(""), _ => SmolStr::new(""),
} }
} }
} }
impl<'a> SubtreeTokenSource<'a> { impl<'a> SubtreeTokenSource {
pub(crate) fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> { pub(crate) fn new(buffer: &TokenBuffer) -> SubtreeTokenSource {
let cursor = buffer.begin(); let mut current = buffer.begin();
let mut cached = Vec::with_capacity(100);
let mut res = SubtreeTokenSource { while !current.eof() {
curr: (Token { kind: EOF, is_jointed_to_next: false }, 0), let cursor = current;
cached_cursor: Cell::new(cursor), let tt = cursor.token_tree();
cached: RefCell::new(Vec::with_capacity(10)),
};
res.curr = (res.mk_token(0), 0);
res
}
fn mk_token(&self, pos: usize) -> Token { // Check if it is lifetime
match *self.get(pos) { if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tt {
Some(ref tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next },
None => Token { kind: EOF, is_jointed_to_next: false },
}
}
fn get(&self, pos: usize) -> Ref<Option<TtToken>> {
fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> {
let tkn = c.token_tree();
if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tkn {
if punct.char == '\'' { if punct.char == '\'' {
let next = c.bump(); let next = cursor.bump();
if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(ident), _)) = if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(ident), _)) =
next.token_tree() next.token_tree()
{ {
let res_cursor = next.bump(); let text = SmolStr::new("'".to_string() + &ident.text);
let text = SmolStr::new("'".to_string() + &ident.to_string()); cached.push(TtToken {
tt: Token { kind: LIFETIME_IDENT, is_jointed_to_next: false },
return Some((res_cursor, text)); text,
});
current = next.bump();
continue;
} else { } else {
panic!("Next token must be ident : {:#?}", next.token_tree()); panic!("Next token must be ident : {:#?}", next.token_tree());
} }
} }
} }
None current = match tt {
}
if pos < self.cached.borrow().len() {
return Ref::map(self.cached.borrow(), |c| &c[pos]);
}
{
let mut cached = self.cached.borrow_mut();
while pos >= cached.len() {
let cursor = self.cached_cursor.get();
if cursor.eof() {
cached.push(None);
continue;
}
if let Some((curr, text)) = is_lifetime(cursor) {
cached.push(Some(TtToken {
kind: LIFETIME_IDENT,
is_joint_to_next: false,
text,
}));
self.cached_cursor.set(curr);
continue;
}
match cursor.token_tree() {
Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => { Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
cached.push(Some(convert_leaf(&leaf))); cached.push(convert_leaf(&leaf));
self.cached_cursor.set(cursor.bump()); cursor.bump()
} }
Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
self.cached_cursor.set(cursor.subtree().unwrap()); cached.push(convert_delim(subtree.delimiter_kind(), false));
cached.push(Some(convert_delim(subtree.delimiter_kind(), false))); cursor.subtree().unwrap()
} }
None => { None => {
if let Some(subtree) = cursor.end() { if let Some(subtree) = cursor.end() {
cached.push(Some(convert_delim(subtree.delimiter_kind(), true))); cached.push(convert_delim(subtree.delimiter_kind(), true));
self.cached_cursor.set(cursor.bump()); cursor.bump()
} else {
continue;
} }
} }
};
}
let mut res = SubtreeTokenSource {
curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
cached,
};
res.curr = (res.token(0), 0);
res
}
fn token(&self, pos: usize) -> Token {
match self.cached.get(pos) {
Some(it) => it.tt,
None => Token { kind: EOF, is_jointed_to_next: false },
} }
} }
} }
Ref::map(self.cached.borrow(), |c| &c[pos]) impl<'a> TokenSource for SubtreeTokenSource {
}
}
impl<'a> TokenSource for SubtreeTokenSource<'a> {
fn current(&self) -> Token { fn current(&self) -> Token {
self.curr.0 self.curr.0
} }
/// Lookahead n token /// Lookahead n token
fn lookahead_nth(&self, n: usize) -> Token { fn lookahead_nth(&self, n: usize) -> Token {
self.mk_token(self.curr.1 + n) self.token(self.curr.1 + n)
} }
/// bump cursor to next token /// bump cursor to next token
@ -133,13 +106,12 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> {
if self.current().kind == EOF { if self.current().kind == EOF {
return; return;
} }
self.curr = (self.token(self.curr.1 + 1), self.curr.1 + 1);
self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1);
} }
/// Is the current token a specified keyword? /// Is the current token a specified keyword?
fn is_keyword(&self, kw: &str) -> bool { fn is_keyword(&self, kw: &str) -> bool {
match *self.get(self.curr.1) { match self.cached.get(self.curr.1) {
Some(ref t) => t.text == *kw, Some(ref t) => t.text == *kw,
_ => false, _ => false,
} }
@ -157,7 +129,7 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
let idx = closing as usize; let idx = closing as usize;
let kind = kinds[idx]; let kind = kinds[idx];
let text = if !texts.is_empty() { &texts[idx..texts.len() - (1 - idx)] } else { "" }; let text = if !texts.is_empty() { &texts[idx..texts.len() - (1 - idx)] } else { "" };
TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) } TtToken { tt: Token { kind, is_jointed_to_next: false }, text: SmolStr::new(text) }
} }
fn convert_literal(l: &tt::Literal) -> TtToken { fn convert_literal(l: &tt::Literal) -> TtToken {
@ -171,7 +143,7 @@ fn convert_literal(l: &tt::Literal) -> TtToken {
}) })
.unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l)); .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
TtToken { kind, is_joint_to_next: false, text: l.text.clone() } TtToken { tt: Token { kind, is_jointed_to_next: false }, text: l.text.clone() }
} }
fn convert_ident(ident: &tt::Ident) -> TtToken { fn convert_ident(ident: &tt::Ident) -> TtToken {
@ -182,7 +154,7 @@ fn convert_ident(ident: &tt::Ident) -> TtToken {
_ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT), _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT),
}; };
TtToken { kind, is_joint_to_next: false, text: ident.text.clone() } TtToken { tt: Token { kind, is_jointed_to_next: false }, text: ident.text.clone() }
} }
fn convert_punct(p: tt::Punct) -> TtToken { fn convert_punct(p: tt::Punct) -> TtToken {
@ -196,7 +168,7 @@ fn convert_punct(p: tt::Punct) -> TtToken {
let s: &str = p.char.encode_utf8(&mut buf); let s: &str = p.char.encode_utf8(&mut buf);
SmolStr::new(s) SmolStr::new(s)
}; };
TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text } TtToken { tt: Token { kind, is_jointed_to_next: p.spacing == tt::Spacing::Joint }, text }
} }
fn convert_leaf(leaf: &tt::Leaf) -> TtToken { fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
@ -210,6 +182,7 @@ fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{convert_literal, TtToken}; use super::{convert_literal, TtToken};
use parser::Token;
use syntax::{SmolStr, SyntaxKind}; use syntax::{SmolStr, SyntaxKind};
#[test] #[test]
@ -220,8 +193,7 @@ mod tests {
text: SmolStr::new("-42.0") text: SmolStr::new("-42.0")
}), }),
TtToken { TtToken {
kind: SyntaxKind::FLOAT_NUMBER, tt: Token { kind: SyntaxKind::FLOAT_NUMBER, is_jointed_to_next: false },
is_joint_to_next: false,
text: SmolStr::new("-42.0") text: SmolStr::new("-42.0")
} }
); );