Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-12-26 04:53:34 +00:00
Code Issues Projects Releases Packages Wiki Activity

converting lexed str to tokens

Browse source
This commit is contained in:
Aleksey Kladov 2021-12-18 15:36:21 +03:00
parent 8b9d145dea
commit 78926027e3
1 changed files with 28 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
crates/parser/src/lexed_str.rs
View file

@ -6,7 +6,7 @@
//! convenient to include a text-based lexer here! //! convenient to include a text-based lexer here!
//! //!
//! Note that these tokens, unlike the tokens we feed into the parser, do //! Note that these tokens, unlike the tokens we feed into the parser, do
//! include info about comments and whitespace. //! include info about comments and whitespace.
use crate::{ use crate::{
SyntaxKind::{self, *}, SyntaxKind::{self, *},
@ -82,18 +82,45 @@ impl<'a> LexedStr<'a> {
assert!(i < self.len()); assert!(i < self.len());
self.kind[i] self.kind[i]
} }
pub fn text(&self, i: usize) -> &str { pub fn text(&self, i: usize) -> &str {
assert!(i < self.len()); assert!(i < self.len());
let lo = self.start[i] as usize; let lo = self.start[i] as usize;
let hi = self.start[i + 1] as usize; let hi = self.start[i + 1] as usize;
&self.text[lo..hi] &self.text[lo..hi]
} }
pub fn error(&self, i: usize) -> Option<&str> { pub fn error(&self, i: usize) -> Option<&str> {
assert!(i < self.len()); assert!(i < self.len());
let err = self.error.binary_search_by_key(&(i as u32), |i| i.token).ok()?; let err = self.error.binary_search_by_key(&(i as u32), |i| i.token).ok()?;
Some(self.error[err].msg.as_str()) Some(self.error[err].msg.as_str())
} }
pub fn to_tokens(&self) -> crate::Tokens {
let mut res = crate::Tokens::default();
let mut was_joint = false;
for i in 0..self.len() {
let kind = self.kind(i);
if kind.is_trivia() {
was_joint = false
} else {
if kind == SyntaxKind::IDENT {
let token_text = self.text(i);
let contextual_kw = SyntaxKind::from_contextual_keyword(token_text)
.unwrap_or(SyntaxKind::IDENT);
res.push_ident(contextual_kw);
} else {
if was_joint {
res.was_joint();
}
res.push(kind);
}
was_joint = true;
}
}
res
}
fn push(&mut self, kind: SyntaxKind, offset: usize) { fn push(&mut self, kind: SyntaxKind, offset: usize) {
self.kind.push(kind); self.kind.push(kind);
self.start.push(offset as u32); self.start.push(offset as u32);