diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index fd447194bf..1e9f59fa53 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -24,40 +24,11 @@ mod tokens; pub(crate) use token_set::TokenSet; -pub use syntax_kind::SyntaxKind; - -use crate::tokens::Tokens; +pub use crate::{syntax_kind::SyntaxKind, tokens::Tokens}; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct ParseError(pub Box); -/// `TokenSource` abstracts the source of the tokens parser operates on. -/// -/// Hopefully this will allow us to treat text and token trees in the same way! -pub trait TokenSource { - fn current(&self) -> Token; - - /// Lookahead n token - fn lookahead_nth(&self, n: usize) -> Token; - - /// bump cursor to next token - fn bump(&mut self); - - /// Is the current token a specified keyword? - fn is_keyword(&self, kw: &str) -> bool; -} - -/// `Token` abstracts the cursor of `TokenSource` operates on. -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub struct Token { - /// What is the current token? - pub kind: SyntaxKind, - - /// Is the current token joined to the next one (`> >` vs `>>`). - pub is_jointed_to_next: bool, - pub contextual_kw: SyntaxKind, -} - /// `TreeSink` abstracts details of a particular syntax tree implementation. pub trait TreeSink { /// Adds new token to the current branch. diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs index 99e7651906..601a5792af 100644 --- a/crates/parser/src/syntax_kind/generated.rs +++ b/crates/parser/src/syntax_kind/generated.rs @@ -334,6 +334,18 @@ impl SyntaxKind { }; Some(kw) } + pub fn from_contextual_keyword(ident: &str) -> Option { + let kw = match ident { + "auto" => AUTO_KW, + "default" => DEFAULT_KW, + "existential" => EXISTENTIAL_KW, + "union" => UNION_KW, + "raw" => RAW_KW, + "macro_rules" => MACRO_RULES_KW, + _ => return None, + }; + Some(kw) + } pub fn from_char(c: char) -> Option { let tok = match c { ';' => SEMICOLON, diff --git a/crates/parser/src/tokens.rs b/crates/parser/src/tokens.rs index 495d9713ea..4f10956070 100644 --- a/crates/parser/src/tokens.rs +++ b/crates/parser/src/tokens.rs @@ -1,8 +1,19 @@ -use crate::{SyntaxKind, Token}; +use crate::SyntaxKind; #[allow(non_camel_case_types)] type bits = u64; +/// `Token` abstracts the cursor of `TokenSource` operates on. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub(crate) struct Token { + /// What is the current token? + pub(crate) kind: SyntaxKind, + + /// Is the current token joined to the next one (`> >` vs `>>`). + pub(crate) is_jointed_to_next: bool, + pub(crate) contextual_kw: SyntaxKind, +} + /// Main input to the parser. /// /// A sequence of tokens represented internally as a struct of arrays. @@ -49,13 +60,14 @@ impl Tokens { self.kind.len() } pub(crate) fn get(&self, idx: usize) -> Token { - if idx > self.len() { - return self.eof(); + if idx < self.len() { + let kind = self.kind[idx]; + let is_jointed_to_next = self.get_joint(idx); + let contextual_kw = self.contextual_kw[idx]; + Token { kind, is_jointed_to_next, contextual_kw } + } else { + self.eof() } - let kind = self.kind[idx]; - let is_jointed_to_next = self.get_joint(idx); - let contextual_kw = self.contextual_kw[idx]; - Token { kind, is_jointed_to_next, contextual_kw } } #[cold] diff --git a/crates/syntax/src/parsing.rs b/crates/syntax/src/parsing.rs index a45f262877..652668e80b 100644 --- a/crates/syntax/src/parsing.rs +++ b/crates/syntax/src/parsing.rs @@ -2,12 +2,10 @@ //! incremental reparsing. pub(crate) mod lexer; -mod text_token_source; mod text_tree_sink; mod reparsing; use parser::SyntaxKind; -use text_token_source::TextTokenSource; use text_tree_sink::TextTreeSink; use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode}; @@ -15,12 +13,12 @@ use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode}; pub(crate) use crate::parsing::{lexer::*, reparsing::incremental_reparse}; pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec) { - let (tokens, lexer_errors) = tokenize(text); + let (lexer_tokens, lexer_errors) = tokenize(text); + let parser_tokens = to_parser_tokens(text, &lexer_tokens); - let mut token_source = TextTokenSource::new(text, &tokens); - let mut tree_sink = TextTreeSink::new(text, &tokens); + let mut tree_sink = TextTreeSink::new(text, &lexer_tokens); - parser::parse_source_file(&mut token_source, &mut tree_sink); + parser::parse_source_file(&parser_tokens, &mut tree_sink); let (tree, mut parser_errors) = tree_sink.finish(); parser_errors.extend(lexer_errors); @@ -33,26 +31,47 @@ pub(crate) fn parse_text_as( text: &str, entry_point: parser::ParserEntryPoint, ) -> Result { - let (tokens, lexer_errors) = tokenize(text); + let (lexer_tokens, lexer_errors) = tokenize(text); if !lexer_errors.is_empty() { return Err(()); } - let mut token_source = TextTokenSource::new(text, &tokens); - let mut tree_sink = TextTreeSink::new(text, &tokens); + let parser_tokens = to_parser_tokens(text, &lexer_tokens); + + let mut tree_sink = TextTreeSink::new(text, &lexer_tokens); // TextTreeSink assumes that there's at least some root node to which it can attach errors and // tokens. We arbitrarily give it a SourceFile. use parser::TreeSink; tree_sink.start_node(SyntaxKind::SOURCE_FILE); - parser::parse(&mut token_source, &mut tree_sink, entry_point); + parser::parse(&parser_tokens, &mut tree_sink, entry_point); tree_sink.finish_node(); - let (tree, parser_errors) = tree_sink.finish(); - use parser::TokenSource; - if !parser_errors.is_empty() || token_source.current().kind != SyntaxKind::EOF { + let (tree, parser_errors, eof) = tree_sink.finish_eof(); + if !parser_errors.is_empty() || !eof { return Err(()); } SyntaxNode::new_root(tree).first_child().and_then(T::cast).ok_or(()) } + +pub(crate) fn to_parser_tokens(text: &str, lexer_tokens: &[lexer::Token]) -> ::parser::Tokens { + let mut off = 0; + let mut res = parser::Tokens::default(); + let mut was_joint = true; + for t in lexer_tokens { + if t.kind.is_trivia() { + was_joint = false; + } else if t.kind == SyntaxKind::IDENT { + let token_text = &text[off..][..usize::from(t.len)]; + let contextual_kw = + SyntaxKind::from_contextual_keyword(token_text).unwrap_or(SyntaxKind::IDENT); + res.push_ident(contextual_kw); + } else { + res.push(was_joint, t.kind); + was_joint = true; + } + off += usize::from(t.len); + } + res +} diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs index 186cc9e74c..62f39a9347 100644 --- a/crates/syntax/src/parsing/reparsing.rs +++ b/crates/syntax/src/parsing/reparsing.rs @@ -12,8 +12,8 @@ use text_edit::Indel; use crate::{ parsing::{ lexer::{lex_single_syntax_kind, tokenize, Token}, - text_token_source::TextTokenSource, text_tree_sink::TextTreeSink, + to_parser_tokens, }, syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode}, SyntaxError, @@ -91,14 +91,14 @@ fn reparse_block( let (node, reparser) = find_reparsable_node(root, edit.delete)?; let text = get_text_after_edit(node.clone().into(), edit); - let (tokens, new_lexer_errors) = tokenize(&text); - if !is_balanced(&tokens) { + let (lexer_tokens, new_lexer_errors) = tokenize(&text); + if !is_balanced(&lexer_tokens) { return None; } + let parser_tokens = to_parser_tokens(&text, &lexer_tokens); - let mut token_source = TextTokenSource::new(&text, &tokens); - let mut tree_sink = TextTreeSink::new(&text, &tokens); - reparser.parse(&mut token_source, &mut tree_sink); + let mut tree_sink = TextTreeSink::new(&text, &lexer_tokens); + reparser.parse(&parser_tokens, &mut tree_sink); let (green, mut new_parser_errors) = tree_sink.finish(); new_parser_errors.extend(new_lexer_errors); diff --git a/crates/syntax/src/parsing/text_token_source.rs b/crates/syntax/src/parsing/text_token_source.rs deleted file mode 100644 index 11dfc63a65..0000000000 --- a/crates/syntax/src/parsing/text_token_source.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! See `TextTokenSource` docs. - -use parser::TokenSource; - -use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextSize}; - -/// Implementation of `parser::TokenSource` that takes tokens from source code text. -pub(crate) struct TextTokenSource<'t> { - text: &'t str, - /// token and its start position (non-whitespace/comment tokens) - /// ```non-rust - /// struct Foo; - /// ^------^--^- - /// | | \________ - /// | \____ \ - /// | \ | - /// (struct, 0) (Foo, 7) (;, 10) - /// ``` - /// `[(struct, 0), (Foo, 7), (;, 10)]` - token_offset_pairs: Vec<(Token, TextSize)>, - - /// Current token and position - curr: (parser::Token, usize), -} - -impl<'t> TokenSource for TextTokenSource<'t> { - fn current(&self) -> parser::Token { - self.curr.0 - } - - fn lookahead_nth(&self, n: usize) -> parser::Token { - mk_token(self.curr.1 + n, &self.token_offset_pairs) - } - - fn bump(&mut self) { - if self.curr.0.kind == EOF { - return; - } - - let pos = self.curr.1 + 1; - self.curr = (mk_token(pos, &self.token_offset_pairs), pos); - } - - fn is_keyword(&self, kw: &str) -> bool { - self.token_offset_pairs - .get(self.curr.1) - .map_or(false, |(token, offset)| &self.text[TextRange::at(*offset, token.len)] == kw) - } -} - -fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> parser::Token { - let (kind, is_jointed_to_next) = match token_offset_pairs.get(pos) { - Some((token, offset)) => ( - token.kind, - token_offset_pairs - .get(pos + 1) - .map_or(false, |(_, next_offset)| offset + token.len == *next_offset), - ), - None => (EOF, false), - }; - parser::Token { kind, is_jointed_to_next } -} - -impl<'t> TextTokenSource<'t> { - /// Generate input from tokens(expect comment and whitespace). - pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> { - let token_offset_pairs: Vec<_> = raw_tokens - .iter() - .filter_map({ - let mut len = 0.into(); - move |token| { - let pair = if token.kind.is_trivia() { None } else { Some((*token, len)) }; - len += token.len; - pair - } - }) - .collect(); - - let first = mk_token(0, &token_offset_pairs); - TextTokenSource { text, token_offset_pairs, curr: (first, 0) } - } -} diff --git a/crates/syntax/src/parsing/text_tree_sink.rs b/crates/syntax/src/parsing/text_tree_sink.rs index 8c1de92048..c1792199fd 100644 --- a/crates/syntax/src/parsing/text_tree_sink.rs +++ b/crates/syntax/src/parsing/text_tree_sink.rs @@ -104,7 +104,7 @@ impl<'a> TextTreeSink<'a> { } } - pub(super) fn finish(mut self) -> (GreenNode, Vec) { + pub(super) fn finish_eof(mut self) -> (GreenNode, Vec, bool) { match mem::replace(&mut self.state, State::Normal) { State::PendingFinish => { self.eat_trivias(); @@ -113,7 +113,15 @@ impl<'a> TextTreeSink<'a> { State::PendingStart | State::Normal => unreachable!(), } - self.inner.finish_raw() + let (node, errors) = self.inner.finish_raw(); + let is_eof = self.token_pos == self.tokens.len(); + + (node, errors, is_eof) + } + + pub(super) fn finish(self) -> (GreenNode, Vec) { + let (node, errors, _eof) = self.finish_eof(); + (node, errors) } fn eat_trivias(&mut self) { diff --git a/crates/syntax/src/tests/sourcegen_ast.rs b/crates/syntax/src/tests/sourcegen_ast.rs index dcd813bbe0..c66edadc3c 100644 --- a/crates/syntax/src/tests/sourcegen_ast.rs +++ b/crates/syntax/src/tests/sourcegen_ast.rs @@ -359,6 +359,10 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String { let full_keywords = full_keywords_values.iter().map(|kw| format_ident!("{}_KW", to_upper_snake_case(kw))); + let contextual_keywords_values = &grammar.contextual_keywords; + let contextual_keywords = + contextual_keywords_values.iter().map(|kw| format_ident!("{}_KW", to_upper_snake_case(kw))); + let all_keywords_values = grammar.keywords.iter().chain(grammar.contextual_keywords.iter()).collect::>(); let all_keywords_idents = all_keywords_values.iter().map(|kw| format_ident!("{}", kw)); @@ -428,6 +432,14 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String { Some(kw) } + pub fn from_contextual_keyword(ident: &str) -> Option { + let kw = match ident { + #(#contextual_keywords_values => #contextual_keywords,)* + _ => return None, + }; + Some(kw) + } + pub fn from_char(c: char) -> Option { let tok = match c { #(#single_byte_tokens_values => #single_byte_tokens,)*