2019-04-07 13:42:53 +00:00
|
|
|
use ra_parser::{TokenSource};
|
2019-05-15 12:35:47 +00:00
|
|
|
use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T};
|
2019-05-22 18:00:34 +00:00
|
|
|
use std::cell::{RefCell, Cell};
|
|
|
|
use tt::buffer::{TokenBuffer, Cursor};
|
2019-04-07 13:42:53 +00:00
|
|
|
|
2019-05-22 18:00:34 +00:00
|
|
|
pub(crate) trait Querier {
|
|
|
|
fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool);
|
2019-04-12 17:50:05 +00:00
|
|
|
}
|
|
|
|
|
2019-04-08 07:58:02 +00:00
|
|
|
#[derive(Debug, Clone, Eq, PartialEq)]
|
2019-04-07 13:42:53 +00:00
|
|
|
struct TtToken {
|
|
|
|
pub kind: SyntaxKind,
|
|
|
|
pub is_joint_to_next: bool,
|
|
|
|
pub text: SmolStr,
|
|
|
|
}
|
|
|
|
|
2019-04-08 07:58:02 +00:00
|
|
|
// A wrapper class for ref cell
|
2019-04-08 10:21:48 +00:00
|
|
|
#[derive(Debug)]
|
2019-05-22 18:00:34 +00:00
|
|
|
pub(crate) struct SubtreeWalk<'a> {
|
|
|
|
start: Cursor<'a>,
|
|
|
|
cursor: Cell<Cursor<'a>>,
|
2019-04-22 14:46:39 +00:00
|
|
|
cached: RefCell<Vec<Option<TtToken>>>,
|
2019-04-08 07:58:02 +00:00
|
|
|
}
|
|
|
|
|
2019-05-22 18:00:34 +00:00
|
|
|
impl<'a> SubtreeWalk<'a> {
|
|
|
|
fn new(cursor: Cursor<'a>) -> Self {
|
|
|
|
SubtreeWalk {
|
|
|
|
start: cursor,
|
|
|
|
cursor: Cell::new(cursor),
|
2019-04-22 14:46:39 +00:00
|
|
|
cached: RefCell::new(Vec::with_capacity(10)),
|
|
|
|
}
|
2019-04-08 12:32:21 +00:00
|
|
|
}
|
|
|
|
|
2019-05-22 18:00:34 +00:00
|
|
|
fn get(&self, pos: usize) -> Option<TtToken> {
|
2019-04-22 14:46:39 +00:00
|
|
|
let mut cached = self.cached.borrow_mut();
|
|
|
|
if pos < cached.len() {
|
|
|
|
return cached[pos].clone();
|
|
|
|
}
|
|
|
|
|
|
|
|
while pos >= cached.len() {
|
2019-05-22 18:00:34 +00:00
|
|
|
let cursor = self.cursor.get();
|
|
|
|
if cursor.eof() {
|
|
|
|
cached.push(None);
|
|
|
|
continue;
|
|
|
|
}
|
2019-05-23 01:31:36 +00:00
|
|
|
|
2019-05-22 18:00:34 +00:00
|
|
|
match cursor.token_tree() {
|
|
|
|
Some(tt::TokenTree::Leaf(leaf)) => {
|
|
|
|
cached.push(Some(convert_leaf(&leaf)));
|
|
|
|
self.cursor.set(cursor.bump());
|
|
|
|
}
|
|
|
|
Some(tt::TokenTree::Subtree(subtree)) => {
|
|
|
|
self.cursor.set(cursor.subtree().unwrap());
|
|
|
|
cached.push(Some(convert_delim(subtree.delimiter, false)));
|
|
|
|
}
|
|
|
|
None => {
|
|
|
|
if let Some(subtree) = cursor.end() {
|
|
|
|
cached.push(Some(convert_delim(subtree.delimiter, true)));
|
|
|
|
self.cursor.set(cursor.bump());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-04-22 14:46:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return cached[pos].clone();
|
2019-04-07 13:42:53 +00:00
|
|
|
}
|
|
|
|
|
2019-05-22 18:00:34 +00:00
|
|
|
fn collect_token_trees(&mut self, n: usize) -> Vec<tt::TokenTree> {
|
|
|
|
let mut res = vec![];
|
2019-05-02 02:19:12 +00:00
|
|
|
|
2019-05-22 18:00:34 +00:00
|
|
|
let mut pos = 0;
|
|
|
|
let mut cursor = self.start;
|
|
|
|
let mut level = 0;
|
2019-04-07 13:42:53 +00:00
|
|
|
|
2019-05-22 18:00:34 +00:00
|
|
|
while pos < n {
|
|
|
|
if cursor.eof() {
|
|
|
|
break;
|
|
|
|
}
|
2019-04-08 07:58:02 +00:00
|
|
|
|
2019-05-22 18:00:34 +00:00
|
|
|
match cursor.token_tree() {
|
|
|
|
Some(tt::TokenTree::Leaf(leaf)) => {
|
|
|
|
if level == 0 {
|
|
|
|
res.push(leaf.into());
|
2019-04-12 17:50:05 +00:00
|
|
|
}
|
2019-05-22 18:00:34 +00:00
|
|
|
cursor = cursor.bump();
|
|
|
|
pos += 1;
|
2019-05-02 02:19:12 +00:00
|
|
|
}
|
2019-05-22 18:00:34 +00:00
|
|
|
Some(tt::TokenTree::Subtree(subtree)) => {
|
|
|
|
if level == 0 {
|
|
|
|
res.push(subtree.into());
|
2019-04-08 07:58:02 +00:00
|
|
|
}
|
2019-05-22 18:00:34 +00:00
|
|
|
pos += 1;
|
|
|
|
level += 1;
|
|
|
|
cursor = cursor.subtree().unwrap();
|
2019-04-08 07:58:02 +00:00
|
|
|
}
|
|
|
|
|
2019-05-22 18:00:34 +00:00
|
|
|
None => {
|
|
|
|
if let Some(_) = cursor.end() {
|
|
|
|
level -= 1;
|
|
|
|
pos += 1;
|
|
|
|
cursor = cursor.bump();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-04-07 13:42:53 +00:00
|
|
|
}
|
|
|
|
|
2019-04-07 16:12:07 +00:00
|
|
|
res
|
|
|
|
}
|
2019-04-08 07:58:02 +00:00
|
|
|
}
|
|
|
|
|
2019-05-22 18:00:34 +00:00
|
|
|
impl<'a> Querier for SubtreeWalk<'a> {
|
2019-04-25 15:12:57 +00:00
|
|
|
fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool) {
|
|
|
|
self.get(uidx)
|
|
|
|
.map(|tkn| (tkn.kind, tkn.text, tkn.is_joint_to_next))
|
|
|
|
.unwrap_or_else(|| (SyntaxKind::EOF, "".into(), false))
|
2019-04-08 07:58:02 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub(crate) struct SubtreeTokenSource<'a> {
|
2019-05-22 18:00:34 +00:00
|
|
|
walker: SubtreeWalk<'a>,
|
2019-04-08 07:58:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl<'a> SubtreeTokenSource<'a> {
|
2019-05-22 18:00:34 +00:00
|
|
|
pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> {
|
|
|
|
SubtreeTokenSource { walker: SubtreeWalk::new(buffer.begin()) }
|
2019-04-08 07:58:02 +00:00
|
|
|
}
|
2019-04-07 16:12:07 +00:00
|
|
|
|
2019-05-22 18:00:34 +00:00
|
|
|
pub fn querier<'b>(&'a self) -> &'b SubtreeWalk<'a>
|
2019-04-08 07:58:02 +00:00
|
|
|
where
|
|
|
|
'a: 'b,
|
|
|
|
{
|
|
|
|
&self.walker
|
2019-04-07 16:12:07 +00:00
|
|
|
}
|
|
|
|
|
2019-05-22 18:00:34 +00:00
|
|
|
pub(crate) fn bump_n(&mut self, parsed_tokens: usize) -> Vec<tt::TokenTree> {
|
2019-04-08 12:32:21 +00:00
|
|
|
let res = self.walker.collect_token_trees(parsed_tokens);
|
2019-04-08 07:58:02 +00:00
|
|
|
res
|
2019-04-07 13:42:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<'a> TokenSource for SubtreeTokenSource<'a> {
|
|
|
|
fn token_kind(&self, pos: usize) -> SyntaxKind {
|
2019-04-08 12:32:21 +00:00
|
|
|
if let Some(tok) = self.walker.get(pos) {
|
2019-04-07 13:42:53 +00:00
|
|
|
tok.kind
|
|
|
|
} else {
|
|
|
|
SyntaxKind::EOF
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fn is_token_joint_to_next(&self, pos: usize) -> bool {
|
2019-04-13 10:38:31 +00:00
|
|
|
match self.walker.get(pos) {
|
|
|
|
Some(t) => t.is_joint_to_next,
|
|
|
|
_ => false,
|
|
|
|
}
|
2019-04-07 13:42:53 +00:00
|
|
|
}
|
|
|
|
fn is_keyword(&self, pos: usize, kw: &str) -> bool {
|
2019-04-13 10:38:31 +00:00
|
|
|
match self.walker.get(pos) {
|
|
|
|
Some(t) => t.text == *kw,
|
|
|
|
_ => false,
|
|
|
|
}
|
2019-04-07 13:42:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-12 17:50:05 +00:00
|
|
|
fn convert_delim(d: tt::Delimiter, closing: bool) -> TtToken {
|
2019-04-08 07:58:02 +00:00
|
|
|
let (kinds, texts) = match d {
|
2019-05-15 12:35:47 +00:00
|
|
|
tt::Delimiter::Parenthesis => ([T!['('], T![')']], "()"),
|
|
|
|
tt::Delimiter::Brace => ([T!['{'], T!['}']], "{}"),
|
|
|
|
tt::Delimiter::Bracket => ([T!['['], T![']']], "[]"),
|
2019-04-12 17:50:05 +00:00
|
|
|
tt::Delimiter::None => ([L_DOLLAR, R_DOLLAR], ""),
|
2019-04-08 07:58:02 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
let idx = closing as usize;
|
|
|
|
let kind = kinds[idx];
|
2019-04-12 17:50:05 +00:00
|
|
|
let text = if texts.len() > 0 { &texts[idx..texts.len() - (1 - idx)] } else { "" };
|
2019-05-02 02:19:12 +00:00
|
|
|
TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) }
|
2019-04-08 07:58:02 +00:00
|
|
|
}
|
2019-04-07 13:42:53 +00:00
|
|
|
|
2019-04-08 07:58:02 +00:00
|
|
|
fn convert_literal(l: &tt::Literal) -> TtToken {
|
2019-04-25 18:56:44 +00:00
|
|
|
let kind =
|
|
|
|
classify_literal(&l.text).map(|tkn| tkn.kind).unwrap_or_else(|| match l.text.as_ref() {
|
2019-05-15 12:35:47 +00:00
|
|
|
"true" => T![true],
|
|
|
|
"false" => T![false],
|
2019-04-25 18:56:44 +00:00
|
|
|
_ => panic!("Fail to convert given literal {:#?}", &l),
|
|
|
|
});
|
2019-04-24 15:01:32 +00:00
|
|
|
|
2019-05-02 02:19:12 +00:00
|
|
|
TtToken { kind, is_joint_to_next: false, text: l.text.clone() }
|
2019-04-08 07:58:02 +00:00
|
|
|
}
|
2019-04-07 13:42:53 +00:00
|
|
|
|
2019-04-08 07:58:02 +00:00
|
|
|
fn convert_ident(ident: &tt::Ident) -> TtToken {
|
2019-04-21 04:32:39 +00:00
|
|
|
let kind = if let Some('\'') = ident.text.chars().next() {
|
|
|
|
LIFETIME
|
|
|
|
} else {
|
|
|
|
SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT)
|
|
|
|
};
|
|
|
|
|
2019-05-02 02:19:12 +00:00
|
|
|
TtToken { kind, is_joint_to_next: false, text: ident.text.clone() }
|
2019-04-08 07:58:02 +00:00
|
|
|
}
|
2019-04-07 13:42:53 +00:00
|
|
|
|
2019-05-02 02:19:12 +00:00
|
|
|
fn convert_punct(p: &tt::Punct) -> TtToken {
|
|
|
|
let kind = match p.char {
|
2019-05-02 13:24:51 +00:00
|
|
|
// lexer may produce compound tokens for these ones
|
2019-05-15 12:35:47 +00:00
|
|
|
'.' => T![.],
|
|
|
|
':' => T![:],
|
|
|
|
'=' => T![=],
|
|
|
|
'!' => T![!],
|
|
|
|
'-' => T![-],
|
2019-05-02 02:19:12 +00:00
|
|
|
c => SyntaxKind::from_char(c).unwrap(),
|
|
|
|
};
|
|
|
|
let text = {
|
|
|
|
let mut buf = [0u8; 4];
|
|
|
|
let s: &str = p.char.encode_utf8(&mut buf);
|
|
|
|
SmolStr::new(s)
|
|
|
|
};
|
|
|
|
TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text }
|
2019-04-07 13:42:53 +00:00
|
|
|
}
|
2019-04-12 17:50:05 +00:00
|
|
|
|
2019-05-02 02:19:12 +00:00
|
|
|
fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
|
2019-04-12 17:50:05 +00:00
|
|
|
match leaf {
|
|
|
|
tt::Leaf::Literal(l) => convert_literal(l),
|
|
|
|
tt::Leaf::Ident(ident) => convert_ident(ident),
|
2019-05-02 02:19:12 +00:00
|
|
|
tt::Leaf::Punct(punct) => convert_punct(punct),
|
2019-04-12 17:50:05 +00:00
|
|
|
}
|
|
|
|
}
|