From 9fd546bec23ac817a45da28889e76118969db91e Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sun, 3 Nov 2019 13:19:50 +0800 Subject: [PATCH 01/16] Add map_id to TokenId --- crates/ra_mbe/src/syntax_bridge.rs | 23 ++++++++++++++++++++--- crates/ra_tt/src/lib.rs | 16 ++++++++++++++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 592fcf527b..8d9217518e 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -12,12 +12,26 @@ use tt::buffer::{Cursor, TokenBuffer}; use crate::subtree_source::SubtreeTokenSource; use crate::ExpandError; +use std::sync::atomic::{AtomicU32, Ordering}; /// Maps `tt::TokenId` to the relative range of the original token. -#[derive(Default)] pub struct TokenMap { /// Maps `tt::TokenId` to the *relative* source range. tokens: Vec, + map_id: u32, +} + +static TOKEN_MAP_COUNTER: AtomicU32 = AtomicU32::new(0); + +/// Generate an unique token map id for each instance +fn make_uniq_token_map_id() -> u32 { + TOKEN_MAP_COUNTER.fetch_add(1, Ordering::SeqCst) +} + +impl std::default::Default for TokenMap { + fn default() -> TokenMap { + TokenMap { tokens: Default::default(), map_id: make_uniq_token_map_id() } + } } /// Convert the syntax tree (what user has written) to a `TokenTree` (what macro @@ -105,14 +119,17 @@ pub fn token_tree_to_items(tt: &tt::Subtree) -> Result, E impl TokenMap { pub fn relative_range_of(&self, tt: tt::TokenId) -> Option { - let idx = tt.0 as usize; + if self.map_id != tt.map_id() { + return None; + } + let idx = tt.token_id() as usize; self.tokens.get(idx).copied() } fn alloc(&mut self, relative_range: TextRange) -> tt::TokenId { let id = self.tokens.len(); self.tokens.push(relative_range); - tt::TokenId(id as u32) + tt::TokenId::new(id as u32, self.map_id) } } diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs index 20c251ff45..96410ff222 100644 --- a/crates/ra_tt/src/lib.rs +++ b/crates/ra_tt/src/lib.rs @@ -25,11 +25,23 @@ use smol_str::SmolStr; /// source token and making sure that identities are preserved during macro /// expansion. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct TokenId(pub u32); +pub struct TokenId(u32, u32); impl TokenId { + pub fn new(token_id: u32, map_id: u32) -> TokenId { + TokenId(token_id, map_id) + } + pub const fn unspecified() -> TokenId { - TokenId(!0) + TokenId(!0, !0) + } + + pub fn token_id(&self) -> u32 { + self.0 + } + + pub fn map_id(&self) -> u32 { + self.1 } } From 159da285e9d8594a2cc4436b5cee7874b07806c9 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sun, 3 Nov 2019 22:46:12 +0800 Subject: [PATCH 02/16] Add macro_expansion_info in hir_expand --- crates/ra_hir_expand/src/db.rs | 93 +++++++++++++++++++----- crates/ra_hir_expand/src/lib.rs | 35 ++++++++- crates/ra_mbe/src/lib.rs | 2 +- crates/ra_mbe/src/syntax_bridge.rs | 113 ++++++++++++++++++++++++----- crates/ra_mbe/src/tests.rs | 16 ++-- 5 files changed, 212 insertions(+), 47 deletions(-) diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs index a4ee9a529a..8abfbb4ffa 100644 --- a/crates/ra_hir_expand/src/db.rs +++ b/crates/ra_hir_expand/src/db.rs @@ -8,10 +8,16 @@ use ra_prof::profile; use ra_syntax::{AstNode, Parse, SyntaxNode}; use crate::{ - ast_id_map::AstIdMap, HirFileId, HirFileIdRepr, MacroCallId, MacroCallLoc, MacroDefId, - MacroFile, MacroFileKind, + ast_id_map::AstIdMap, ExpansionInfo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallLoc, + MacroDefId, MacroFile, MacroFileKind, }; +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct ParseMacroWithInfo { + pub parsed: Parse, + pub expansion_info: Arc, +} + // FIXME: rename to ExpandDatabase #[salsa::query_group(AstDatabaseStorage)] pub trait AstDatabase: SourceDatabase { @@ -22,10 +28,16 @@ pub trait AstDatabase: SourceDatabase { #[salsa::interned] fn intern_macro(&self, macro_call: MacroCallLoc) -> MacroCallId; - fn macro_arg(&self, id: MacroCallId) -> Option>; - fn macro_def(&self, id: MacroDefId) -> Option>; + fn macro_arg(&self, id: MacroCallId) -> Option<(Arc, Arc)>; + fn macro_def(&self, id: MacroDefId) -> Option<(Arc, Arc)>; fn parse_macro(&self, macro_file: MacroFile) -> Option>; - fn macro_expand(&self, macro_call: MacroCallId) -> Result, String>; + fn parse_macro_with_info(&self, macro_file: MacroFile) -> Option; + fn macro_expand( + &self, + macro_call: MacroCallId, + ) -> Result<(Arc, (Arc, Arc)), String>; + + fn macro_expansion_info(&self, macro_file: MacroFile) -> Option>; } pub(crate) fn ast_id_map(db: &dyn AstDatabase, file_id: HirFileId) -> Arc { @@ -34,10 +46,13 @@ pub(crate) fn ast_id_map(db: &dyn AstDatabase, file_id: HirFileId) -> Arc Option> { +pub(crate) fn macro_def( + db: &dyn AstDatabase, + id: MacroDefId, +) -> Option<(Arc, Arc)> { let macro_call = id.ast_id.to_node(db); let arg = macro_call.token_tree()?; - let (tt, _) = mbe::ast_to_token_tree(&arg).or_else(|| { + let (tt, tmap) = mbe::ast_to_token_tree(&arg).or_else(|| { log::warn!("fail on macro_def to token tree: {:#?}", arg); None })?; @@ -45,32 +60,36 @@ pub(crate) fn macro_def(db: &dyn AstDatabase, id: MacroDefId) -> Option Option> { +pub(crate) fn macro_arg( + db: &dyn AstDatabase, + id: MacroCallId, +) -> Option<(Arc, Arc)> { let loc = db.lookup_intern_macro(id); let macro_call = loc.ast_id.to_node(db); let arg = macro_call.token_tree()?; - let (tt, _) = mbe::ast_to_token_tree(&arg)?; - Some(Arc::new(tt)) + let (tt, tmap) = mbe::ast_to_token_tree(&arg)?; + Some((Arc::new(tt), Arc::new(tmap))) } pub(crate) fn macro_expand( db: &dyn AstDatabase, id: MacroCallId, -) -> Result, String> { +) -> Result<(Arc, (Arc, Arc)), String> { let loc = db.lookup_intern_macro(id); let macro_arg = db.macro_arg(id).ok_or("Fail to args in to tt::TokenTree")?; let macro_rules = db.macro_def(loc.def).ok_or("Fail to find macro definition")?; - let tt = macro_rules.expand(¯o_arg).map_err(|err| format!("{:?}", err))?; + let tt = macro_rules.0.expand(¯o_arg.0).map_err(|err| format!("{:?}", err))?; // Set a hard limit for the expanded tt let count = tt.count(); if count > 65536 { return Err(format!("Total tokens count exceed limit : count = {}", count)); } - Ok(Arc::new(tt)) + + Ok((Arc::new(tt), (macro_arg.1.clone(), macro_rules.1.clone()))) } pub(crate) fn parse_or_expand(db: &dyn AstDatabase, file_id: HirFileId) -> Option { @@ -87,6 +106,13 @@ pub(crate) fn parse_macro( macro_file: MacroFile, ) -> Option> { let _p = profile("parse_macro_query"); + db.parse_macro_with_info(macro_file).map(|r| r.parsed) +} + +pub(crate) fn parse_macro_with_info( + db: &dyn AstDatabase, + macro_file: MacroFile, +) -> Option { let macro_call_id = macro_file.macro_call_id; let tt = db .macro_expand(macro_call_id) @@ -97,8 +123,39 @@ pub(crate) fn parse_macro( log::warn!("fail on macro_parse: (reason: {})", err,); }) .ok()?; - match macro_file.macro_file_kind { - MacroFileKind::Items => mbe::token_tree_to_items(&tt).ok().map(Parse::to_syntax), - MacroFileKind::Expr => mbe::token_tree_to_expr(&tt).ok().map(Parse::to_syntax), - } + let res = match macro_file.macro_file_kind { + MacroFileKind::Items => { + mbe::token_tree_to_items(&tt.0).ok().map(|(p, map)| (Parse::to_syntax(p), map)) + } + MacroFileKind::Expr => { + mbe::token_tree_to_expr(&tt.0).ok().map(|(p, map)| (Parse::to_syntax(p), map)) + } + }; + + res.map(|(parsed, exp_map)| { + let (arg_map, def_map) = tt.1; + let loc: MacroCallLoc = db.lookup_intern_macro(macro_call_id); + + let def_start = + loc.def.ast_id.to_node(db).token_tree().map(|t| t.syntax().text_range().start()); + let arg_start = + loc.ast_id.to_node(db).token_tree().map(|t| t.syntax().text_range().start()); + + let arg_map = + arg_start.map(|start| exp_map.ranges(&arg_map, start)).unwrap_or_else(|| Vec::new()); + + let def_map = + def_start.map(|start| exp_map.ranges(&def_map, start)).unwrap_or_else(|| Vec::new()); + + let info = ExpansionInfo { arg_map, def_map }; + + ParseMacroWithInfo { parsed, expansion_info: Arc::new(info) } + }) +} + +pub(crate) fn macro_expansion_info( + db: &dyn AstDatabase, + macro_file: MacroFile, +) -> Option> { + db.parse_macro_with_info(macro_file).map(|res| res.expansion_info.clone()) } diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs index dd07a16b4c..194020b45c 100644 --- a/crates/ra_hir_expand/src/lib.rs +++ b/crates/ra_hir_expand/src/lib.rs @@ -16,7 +16,7 @@ use std::hash::{Hash, Hasher}; use ra_db::{salsa, CrateId, FileId}; use ra_syntax::{ ast::{self, AstNode}, - SyntaxNode, + SyntaxNode, TextRange, }; use crate::ast_id_map::FileAstId; @@ -112,6 +112,39 @@ impl MacroCallId { } } +#[derive(Debug, Clone, PartialEq, Eq)] +/// ExpansionInfo mainly describle how to map text range between src and expaned macro +pub struct ExpansionInfo { + pub arg_map: Vec<(TextRange, TextRange)>, + pub def_map: Vec<(TextRange, TextRange)>, +} + +impl ExpansionInfo { + pub fn find_range( + &self, + from: TextRange, + (arg_file_id, def_file_id): (HirFileId, HirFileId), + ) -> Option<(HirFileId, TextRange)> { + for (src, dest) in &self.arg_map { + dbg!((src, *dest, "arg_map")); + if src.is_subrange(&from) { + dbg!((arg_file_id, *dest)); + return Some((arg_file_id, *dest)); + } + } + + for (src, dest) in &self.def_map { + dbg!((src, *dest, "def_map")); + if src.is_subrange(&from) { + dbg!((arg_file_id, *dest)); + return Some((def_file_id, *dest)); + } + } + + None + } +} + /// `AstId` points to an AST node in any file. /// /// It is stable across reparses, and can be used as salsa key/value. diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 15f0001755..2926b29fd0 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -32,7 +32,7 @@ pub enum ExpandError { pub use crate::syntax_bridge::{ ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_expr, token_tree_to_items, - token_tree_to_macro_stmts, token_tree_to_pat, token_tree_to_ty, + token_tree_to_macro_stmts, token_tree_to_pat, token_tree_to_ty, TokenMap, }; /// This struct contains AST for a single `macro_rules` definition. What might diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 8d9217518e..5db6647e3a 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -15,6 +15,7 @@ use crate::ExpandError; use std::sync::atomic::{AtomicU32, Ordering}; /// Maps `tt::TokenId` to the relative range of the original token. +#[derive(Debug, PartialEq, Eq)] pub struct TokenMap { /// Maps `tt::TokenId` to the *relative* source range. tokens: Vec, @@ -34,6 +35,13 @@ impl std::default::Default for TokenMap { } } +/// Maps Relative range of the expanded syntax node to `tt::TokenId` +#[derive(Debug, PartialEq, Eq, Default)] +pub struct ExpandedRangeMap { + /// Maps `tt::TokenId` to the *relative* source range. + ranges: Vec<(TextRange, tt::TokenId)>, +} + /// Convert the syntax tree (what user has written) to a `TokenTree` (what macro /// will consume). pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap)> { @@ -66,7 +74,7 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke fn fragment_to_syntax_node( tt: &tt::Subtree, fragment_kind: FragmentKind, -) -> Result, ExpandError> { +) -> Result<(Parse, ExpandedRangeMap), ExpandError> { let tmp; let tokens = match tt { tt::Subtree { delimiter: tt::Delimiter::None, token_trees } => token_trees.as_slice(), @@ -77,44 +85,55 @@ fn fragment_to_syntax_node( }; let buffer = TokenBuffer::new(&tokens); let mut token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(buffer.begin()); + let mut range_map = ExpandedRangeMap::default(); + let mut tree_sink = TtTreeSink::new(buffer.begin(), &mut range_map); ra_parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } //FIXME: would be cool to report errors let parse = tree_sink.inner.finish(); - Ok(parse) + Ok((parse, range_map)) } /// Parses the token tree (result of macro expansion) to an expression -pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, ExpandError> { - let parse = fragment_to_syntax_node(tt, Expr)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError) +pub fn token_tree_to_expr( + tt: &tt::Subtree, +) -> Result<(Parse, ExpandedRangeMap), ExpandError> { + let (parse, map) = fragment_to_syntax_node(tt, Expr)?; + parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) } /// Parses the token tree (result of macro expansion) to a Pattern -pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result, ExpandError> { - let parse = fragment_to_syntax_node(tt, Pattern)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError) +pub fn token_tree_to_pat( + tt: &tt::Subtree, +) -> Result<(Parse, ExpandedRangeMap), ExpandError> { + let (parse, map) = fragment_to_syntax_node(tt, Pattern)?; + parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) } /// Parses the token tree (result of macro expansion) to a Type -pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result, ExpandError> { - let parse = fragment_to_syntax_node(tt, Type)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError) +pub fn token_tree_to_ty( + tt: &tt::Subtree, +) -> Result<(Parse, ExpandedRangeMap), ExpandError> { + let (parse, map) = fragment_to_syntax_node(tt, Type)?; + parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) } /// Parses the token tree (result of macro expansion) as a sequence of stmts -pub fn token_tree_to_macro_stmts(tt: &tt::Subtree) -> Result, ExpandError> { - let parse = fragment_to_syntax_node(tt, Statements)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError) +pub fn token_tree_to_macro_stmts( + tt: &tt::Subtree, +) -> Result<(Parse, ExpandedRangeMap), ExpandError> { + let (parse, map) = fragment_to_syntax_node(tt, Statements)?; + parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) } /// Parses the token tree (result of macro expansion) as a sequence of items -pub fn token_tree_to_items(tt: &tt::Subtree) -> Result, ExpandError> { - let parse = fragment_to_syntax_node(tt, Items)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError) +pub fn token_tree_to_items( + tt: &tt::Subtree, +) -> Result<(Parse, ExpandedRangeMap), ExpandError> { + let (parse, map) = fragment_to_syntax_node(tt, Items)?; + parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) } impl TokenMap { @@ -133,6 +152,28 @@ impl TokenMap { } } +impl ExpandedRangeMap { + fn set(&mut self, relative_range: TextRange, token_id: &tt::TokenId) { + self.ranges.push((relative_range, token_id.clone())) + } + + pub fn ranges(&self, to: &TokenMap) -> Vec<(TextRange, TextRange)> { + self.ranges + .iter() + .filter_map(|(r, tid)| { + if to.map_id == tid.map_id() { + return None; + } + if let Some(to_range) = to.relative_range_of(*tid) { + Some((*r, to_range)) + } else { + None + } + }) + .collect() + } +} + /// Returns the textual content of a doc comment block as a quoted string /// That is, strips leading `///` (or `/**`, etc) /// and strips the ending `*/` @@ -279,6 +320,8 @@ struct TtTreeSink<'a> { cursor: Cursor<'a>, text_pos: TextUnit, inner: SyntaxTreeBuilder, + range_marker: Option<(TextRange, tt::TokenId)>, + range_map: &'a mut ExpandedRangeMap, // Number of roots // Use for detect ill-form tree which is not single root @@ -286,13 +329,15 @@ struct TtTreeSink<'a> { } impl<'a> TtTreeSink<'a> { - fn new(cursor: Cursor<'a>) -> Self { + fn new(cursor: Cursor<'a>, range_map: &'a mut ExpandedRangeMap) -> Self { TtTreeSink { buf: String::new(), cursor, text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), roots: smallvec::SmallVec::new(), + range_map, + range_marker: None, } } } @@ -317,6 +362,8 @@ impl<'a> TreeSink for TtTreeSink<'a> { return; } + let mut last_ident = None; + for _ in 0..n_tokens { if self.cursor.eof() { break; @@ -326,6 +373,10 @@ impl<'a> TreeSink for TtTreeSink<'a> { Some(tt::TokenTree::Leaf(leaf)) => { self.cursor = self.cursor.bump(); self.buf += &format!("{}", leaf); + + if let tt::Leaf::Ident(ident) = leaf { + last_ident = Some(ident); + } } Some(tt::TokenTree::Subtree(subtree)) => { self.cursor = self.cursor.subtree().unwrap(); @@ -345,6 +396,14 @@ impl<'a> TreeSink for TtTreeSink<'a> { self.buf.clear(); self.inner.token(kind, text); + // Mark the range if needed + if let Some((range, token_id)) = self.range_marker.as_mut() { + if let Some(ident) = last_ident { + *range = TextRange::offset_len(range.start(), TextUnit::of_str(&ident.text)); + *token_id = ident.id; + } + } + // Add whitespace between adjoint puncts let next = self.cursor.bump(); if let ( @@ -354,6 +413,7 @@ impl<'a> TreeSink for TtTreeSink<'a> { { if curr.spacing == tt::Spacing::Alone { self.inner.token(WHITESPACE, " ".into()); + self.text_pos += TextUnit::of_char(' '); } } } @@ -361,6 +421,15 @@ impl<'a> TreeSink for TtTreeSink<'a> { fn start_node(&mut self, kind: SyntaxKind) { self.inner.start_node(kind); + self.range_marker = if kind == IDENT { + Some(( + TextRange::offset_len(self.text_pos, TextUnit::from_usize(0)), + tt::TokenId::unspecified(), + )) + } else { + None + }; + match self.roots.last_mut() { None | Some(0) => self.roots.push(1), Some(ref mut n) => **n += 1, @@ -370,6 +439,12 @@ impl<'a> TreeSink for TtTreeSink<'a> { fn finish_node(&mut self) { self.inner.finish_node(); *self.roots.last_mut().unwrap() -= 1; + + if let Some(range) = self.range_marker { + if range.1 != tt::TokenId::unspecified() { + self.range_map.set(range.0, &range.1) + } + } } fn error(&mut self, error: ParseError) { diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index a23e3afe3e..a848ea3347 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -126,7 +126,7 @@ fn test_expr_order() { "#, ); let expanded = expand(&rules, "foo! { 1 + 1}"); - let tree = token_tree_to_items(&expanded).unwrap().tree(); + let tree = token_tree_to_items(&expanded).unwrap().0.tree(); let dump = format!("{:#?}", tree.syntax()); assert_eq_text!( @@ -383,7 +383,7 @@ fn test_expand_to_item_list() { ", ); let expansion = expand(&rules, "structs!(Foo, Bar);"); - let tree = token_tree_to_items(&expansion).unwrap().tree(); + let tree = token_tree_to_items(&expansion).unwrap().0.tree(); assert_eq!( format!("{:#?}", tree.syntax()).trim(), r#" @@ -501,7 +501,7 @@ fn test_tt_to_stmts() { ); let expanded = expand(&rules, "foo!{}"); - let stmts = token_tree_to_macro_stmts(&expanded).unwrap().tree(); + let stmts = token_tree_to_macro_stmts(&expanded).unwrap().0.tree(); assert_eq!( format!("{:#?}", stmts.syntax()).trim(), @@ -946,7 +946,7 @@ fn test_vec() { ); let expansion = expand(&rules, r#"vec![1u32,2];"#); - let tree = token_tree_to_expr(&expansion).unwrap().tree(); + let tree = token_tree_to_expr(&expansion).unwrap().0.tree(); assert_eq!( format!("{:#?}", tree.syntax()).trim(), @@ -1436,8 +1436,8 @@ pub(crate) fn assert_expansion( }; let (expanded_tree, expected_tree) = match kind { MacroKind::Items => { - let expanded_tree = token_tree_to_items(&expanded).unwrap().tree(); - let expected_tree = token_tree_to_items(&expected).unwrap().tree(); + let expanded_tree = token_tree_to_items(&expanded).unwrap().0.tree(); + let expected_tree = token_tree_to_items(&expected).unwrap().0.tree(); ( debug_dump_ignore_spaces(expanded_tree.syntax()).trim().to_string(), @@ -1446,8 +1446,8 @@ pub(crate) fn assert_expansion( } MacroKind::Stmts => { - let expanded_tree = token_tree_to_macro_stmts(&expanded).unwrap().tree(); - let expected_tree = token_tree_to_macro_stmts(&expected).unwrap().tree(); + let expanded_tree = token_tree_to_macro_stmts(&expanded).unwrap().0.tree(); + let expected_tree = token_tree_to_macro_stmts(&expected).unwrap().0.tree(); ( debug_dump_ignore_spaces(expanded_tree.syntax()).trim().to_string(), From ae609d7953297b355616c7862b8deefe74a8f95f Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 4 Nov 2019 01:44:23 +0800 Subject: [PATCH 03/16] Add parent_expansion to HirFileId --- crates/ra_hir_expand/src/lib.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs index 194020b45c..180cd1f572 100644 --- a/crates/ra_hir_expand/src/lib.rs +++ b/crates/ra_hir_expand/src/lib.rs @@ -20,6 +20,7 @@ use ra_syntax::{ }; use crate::ast_id_map::FileAstId; +use std::sync::Arc; /// Input to the analyzer is a set of files, where each file is identified by /// `FileId` and contains source code. However, another source of source code in @@ -66,6 +67,24 @@ impl HirFileId { } } } + + /// Return expansion information if it is a macro-expansion file + pub fn parent_expansion( + self, + db: &dyn db::AstDatabase, + ) -> Option<((HirFileId, HirFileId), Arc)> { + match self.0 { + HirFileIdRepr::FileId(_) => None, + HirFileIdRepr::MacroFile(macro_file) => { + let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id); + + let def_file = loc.def.ast_id.file_id; + let arg_file = loc.ast_id.file_id; + + db.macro_expansion_info(macro_file).map(|ex| ((arg_file, def_file), ex)) + } + } + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] From d560219d06780ea4bd3cca20e1d3883f2d68d59a Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 4 Nov 2019 01:45:58 +0800 Subject: [PATCH 04/16] Fixed bug in ExpandedRangeMap --- crates/ra_mbe/src/syntax_bridge.rs | 50 ++++++++++-------------------- 1 file changed, 16 insertions(+), 34 deletions(-) diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 5db6647e3a..4965f9c33b 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -157,15 +157,16 @@ impl ExpandedRangeMap { self.ranges.push((relative_range, token_id.clone())) } - pub fn ranges(&self, to: &TokenMap) -> Vec<(TextRange, TextRange)> { + pub fn ranges(&self, to: &TokenMap, start: TextUnit) -> Vec<(TextRange, TextRange)> { + dbg!(&self.ranges); self.ranges .iter() .filter_map(|(r, tid)| { - if to.map_id == tid.map_id() { + if to.map_id != tid.map_id() { return None; } if let Some(to_range) = to.relative_range_of(*tid) { - Some((*r, to_range)) + Some((*r, TextRange::from_to(to_range.start() + start, to_range.end() + start))) } else { None } @@ -320,7 +321,6 @@ struct TtTreeSink<'a> { cursor: Cursor<'a>, text_pos: TextUnit, inner: SyntaxTreeBuilder, - range_marker: Option<(TextRange, tt::TokenId)>, range_map: &'a mut ExpandedRangeMap, // Number of roots @@ -337,7 +337,6 @@ impl<'a> TtTreeSink<'a> { inner: SyntaxTreeBuilder::default(), roots: smallvec::SmallVec::new(), range_map, - range_marker: None, } } } @@ -362,8 +361,6 @@ impl<'a> TreeSink for TtTreeSink<'a> { return; } - let mut last_ident = None; - for _ in 0..n_tokens { if self.cursor.eof() { break; @@ -371,12 +368,20 @@ impl<'a> TreeSink for TtTreeSink<'a> { match self.cursor.token_tree() { Some(tt::TokenTree::Leaf(leaf)) => { + // Mark the range if needed + if let tt::Leaf::Ident(ident) = leaf { + if kind == IDENT { + let range = TextRange::offset_len( + self.text_pos + TextUnit::of_str(&self.buf), + TextUnit::of_str(&ident.text), + ); + let token_id = ident.id; + self.range_map.set(range, &token_id); + } + } + self.cursor = self.cursor.bump(); self.buf += &format!("{}", leaf); - - if let tt::Leaf::Ident(ident) = leaf { - last_ident = Some(ident); - } } Some(tt::TokenTree::Subtree(subtree)) => { self.cursor = self.cursor.subtree().unwrap(); @@ -396,14 +401,6 @@ impl<'a> TreeSink for TtTreeSink<'a> { self.buf.clear(); self.inner.token(kind, text); - // Mark the range if needed - if let Some((range, token_id)) = self.range_marker.as_mut() { - if let Some(ident) = last_ident { - *range = TextRange::offset_len(range.start(), TextUnit::of_str(&ident.text)); - *token_id = ident.id; - } - } - // Add whitespace between adjoint puncts let next = self.cursor.bump(); if let ( @@ -421,15 +418,6 @@ impl<'a> TreeSink for TtTreeSink<'a> { fn start_node(&mut self, kind: SyntaxKind) { self.inner.start_node(kind); - self.range_marker = if kind == IDENT { - Some(( - TextRange::offset_len(self.text_pos, TextUnit::from_usize(0)), - tt::TokenId::unspecified(), - )) - } else { - None - }; - match self.roots.last_mut() { None | Some(0) => self.roots.push(1), Some(ref mut n) => **n += 1, @@ -439,12 +427,6 @@ impl<'a> TreeSink for TtTreeSink<'a> { fn finish_node(&mut self) { self.inner.finish_node(); *self.roots.last_mut().unwrap() -= 1; - - if let Some(range) = self.range_marker { - if range.1 != tt::TokenId::unspecified() { - self.range_map.set(range.0, &range.1) - } - } } fn error(&mut self, error: ParseError) { From 67226ebc8288b9489867ea2454f9d976b65ff15e Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 4 Nov 2019 01:47:59 +0800 Subject: [PATCH 05/16] Remove dbg! --- crates/ra_hir_expand/src/lib.rs | 4 ---- crates/ra_mbe/src/syntax_bridge.rs | 1 - 2 files changed, 5 deletions(-) diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs index 180cd1f572..b6e55aa6ad 100644 --- a/crates/ra_hir_expand/src/lib.rs +++ b/crates/ra_hir_expand/src/lib.rs @@ -145,17 +145,13 @@ impl ExpansionInfo { (arg_file_id, def_file_id): (HirFileId, HirFileId), ) -> Option<(HirFileId, TextRange)> { for (src, dest) in &self.arg_map { - dbg!((src, *dest, "arg_map")); if src.is_subrange(&from) { - dbg!((arg_file_id, *dest)); return Some((arg_file_id, *dest)); } } for (src, dest) in &self.def_map { - dbg!((src, *dest, "def_map")); if src.is_subrange(&from) { - dbg!((arg_file_id, *dest)); return Some((def_file_id, *dest)); } } diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 4965f9c33b..8e86742d03 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -158,7 +158,6 @@ impl ExpandedRangeMap { } pub fn ranges(&self, to: &TokenMap, start: TextUnit) -> Vec<(TextRange, TextRange)> { - dbg!(&self.ranges); self.ranges .iter() .filter_map(|(r, tid)| { From e8741b9d75eb4edd711c6516c3b17c87c0431166 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 4 Nov 2019 01:49:41 +0800 Subject: [PATCH 06/16] Use new expansion feature in goto_definition --- .../src/display/navigation_target.rs | 122 +++++++++++++----- crates/ra_ide_api/src/goto_definition.rs | 37 ++++-- 2 files changed, 114 insertions(+), 45 deletions(-) diff --git a/crates/ra_ide_api/src/display/navigation_target.rs b/crates/ra_ide_api/src/display/navigation_target.rs index 5cb67fb95b..0c3e25ce67 100644 --- a/crates/ra_ide_api/src/display/navigation_target.rs +++ b/crates/ra_ide_api/src/display/navigation_target.rs @@ -29,6 +29,20 @@ pub struct NavigationTarget { docs: Option, } +fn find_range_from_node( + db: &RootDatabase, + src: hir::HirFileId, + node: &SyntaxNode, +) -> (FileId, TextRange) { + let text_range = node.text_range(); + let (file_id, text_range) = src + .parent_expansion(db) + .and_then(|(files, expansion_info)| expansion_info.find_range(text_range, files)) + .unwrap_or((src, text_range)); + + (file_id.original_file(db), text_range) +} + impl NavigationTarget { /// When `focus_range` is specified, returns it. otherwise /// returns `full_range` @@ -72,8 +86,12 @@ impl NavigationTarget { self.focus_range } - pub(crate) fn from_bind_pat(file_id: FileId, pat: &ast::BindPat) -> NavigationTarget { - NavigationTarget::from_named(file_id, pat, None, None) + pub(crate) fn from_bind_pat( + db: &RootDatabase, + file_id: FileId, + pat: &ast::BindPat, + ) -> NavigationTarget { + NavigationTarget::from_named(db, file_id.into(), pat, None, None) } pub(crate) fn from_symbol(db: &RootDatabase, symbol: FileSymbol) -> NavigationTarget { @@ -96,7 +114,7 @@ impl NavigationTarget { ) -> NavigationTarget { let parse = db.parse(file_id); let pat = pat.to_node(parse.tree().syntax()); - NavigationTarget::from_bind_pat(file_id, &pat) + NavigationTarget::from_bind_pat(db, file_id, &pat) } pub(crate) fn from_self_param( @@ -119,31 +137,47 @@ impl NavigationTarget { pub(crate) fn from_module(db: &RootDatabase, module: hir::Module) -> NavigationTarget { let src = module.definition_source(db); - let file_id = src.file_id.original_file(db); let name = module.name(db).map(|it| it.to_string().into()).unwrap_or_default(); + match src.ast { ModuleSource::SourceFile(node) => { - NavigationTarget::from_syntax(file_id, name, None, node.syntax(), None, None) + let (file_id, text_range) = find_range_from_node(db, src.file_id, node.syntax()); + + NavigationTarget::from_syntax( + file_id, + name, + None, + text_range, + node.syntax(), + None, + None, + ) + } + ModuleSource::Module(node) => { + let (file_id, text_range) = find_range_from_node(db, src.file_id, node.syntax()); + + NavigationTarget::from_syntax( + file_id, + name, + None, + text_range, + node.syntax(), + node.doc_comment_text(), + node.short_label(), + ) } - ModuleSource::Module(node) => NavigationTarget::from_syntax( - file_id, - name, - None, - node.syntax(), - node.doc_comment_text(), - node.short_label(), - ), } } pub(crate) fn from_module_to_decl(db: &RootDatabase, module: hir::Module) -> NavigationTarget { let name = module.name(db).map(|it| it.to_string().into()).unwrap_or_default(); if let Some(src) = module.declaration_source(db) { - let file_id = src.file_id.original_file(db); + let (file_id, text_range) = find_range_from_node(db, src.file_id, src.ast.syntax()); return NavigationTarget::from_syntax( file_id, name, None, + text_range, src.ast.syntax(), src.ast.doc_comment_text(), src.ast.short_label(), @@ -154,13 +188,25 @@ impl NavigationTarget { pub(crate) fn from_field(db: &RootDatabase, field: hir::StructField) -> NavigationTarget { let src = field.source(db); - let file_id = src.file_id.original_file(db); match src.ast { - FieldSource::Named(it) => { - NavigationTarget::from_named(file_id, &it, it.doc_comment_text(), it.short_label()) - } + FieldSource::Named(it) => NavigationTarget::from_named( + db, + src.file_id, + &it, + it.doc_comment_text(), + it.short_label(), + ), FieldSource::Pos(it) => { - NavigationTarget::from_syntax(file_id, "".into(), None, it.syntax(), None, None) + let (file_id, text_range) = find_range_from_node(db, src.file_id, it.syntax()); + NavigationTarget::from_syntax( + file_id, + "".into(), + None, + text_range, + it.syntax(), + None, + None, + ) } } } @@ -172,7 +218,8 @@ impl NavigationTarget { { let src = def.source(db); NavigationTarget::from_named( - src.file_id.original_file(db), + db, + src.file_id, &src.ast, src.ast.doc_comment_text(), src.ast.short_label(), @@ -212,10 +259,13 @@ impl NavigationTarget { impl_block: hir::ImplBlock, ) -> NavigationTarget { let src = impl_block.source(db); + let (file_id, text_range) = find_range_from_node(db, src.file_id, src.ast.syntax()); + NavigationTarget::from_syntax( - src.file_id.original_file(db), + file_id, "impl".into(), None, + text_range, src.ast.syntax(), None, None, @@ -236,12 +286,7 @@ impl NavigationTarget { pub(crate) fn from_macro_def(db: &RootDatabase, macro_call: hir::MacroDef) -> NavigationTarget { let src = macro_call.source(db); log::debug!("nav target {:#?}", src.ast.syntax()); - NavigationTarget::from_named( - src.file_id.original_file(db), - &src.ast, - src.ast.doc_comment_text(), - None, - ) + NavigationTarget::from_named(db, src.file_id, &src.ast, src.ast.doc_comment_text(), None) } #[cfg(test)] @@ -270,21 +315,35 @@ impl NavigationTarget { /// Allows `NavigationTarget` to be created from a `NameOwner` pub(crate) fn from_named( - file_id: FileId, + db: &RootDatabase, + file_id: hir::HirFileId, node: &impl ast::NameOwner, docs: Option, description: Option, ) -> NavigationTarget { //FIXME: use `_` instead of empty string let name = node.name().map(|it| it.text().clone()).unwrap_or_default(); - let focus_range = node.name().map(|it| it.syntax().text_range()); - NavigationTarget::from_syntax(file_id, name, focus_range, node.syntax(), docs, description) + + let focus_range = node.name().map(|it| find_range_from_node(db, file_id, it.syntax()).1); + + let (file_id, full_range) = find_range_from_node(db, file_id, node.syntax()); + + NavigationTarget::from_syntax( + file_id, + name, + focus_range, + full_range, + node.syntax(), + docs, + description, + ) } fn from_syntax( file_id: FileId, name: SmolStr, focus_range: Option, + full_range: TextRange, node: &SyntaxNode, docs: Option, description: Option, @@ -293,9 +352,8 @@ impl NavigationTarget { file_id, name, kind: node.kind(), - full_range: node.text_range(), + full_range, focus_range, - // ptr: Some(LocalSyntaxPtr::new(node)), container_name: None, description, docs, diff --git a/crates/ra_ide_api/src/goto_definition.rs b/crates/ra_ide_api/src/goto_definition.rs index c1ce54bea3..97127706f3 100644 --- a/crates/ra_ide_api/src/goto_definition.rs +++ b/crates/ra_ide_api/src/goto_definition.rs @@ -101,19 +101,20 @@ pub(crate) fn name_definition( } } - if let Some(nav) = named_target(file_id, &parent) { + if let Some(nav) = named_target(db, file_id, &parent) { return Some(vec![nav]); } None } -fn named_target(file_id: FileId, node: &SyntaxNode) -> Option { +fn named_target(db: &RootDatabase, file_id: FileId, node: &SyntaxNode) -> Option { match_ast! { match node { ast::StructDef(it) => { Some(NavigationTarget::from_named( - file_id, + db, + file_id.into(), &it, it.doc_comment_text(), it.short_label(), @@ -121,7 +122,8 @@ fn named_target(file_id: FileId, node: &SyntaxNode) -> Option }, ast::EnumDef(it) => { Some(NavigationTarget::from_named( - file_id, + db, + file_id.into(), &it, it.doc_comment_text(), it.short_label(), @@ -129,7 +131,8 @@ fn named_target(file_id: FileId, node: &SyntaxNode) -> Option }, ast::EnumVariant(it) => { Some(NavigationTarget::from_named( - file_id, + db, + file_id.into(), &it, it.doc_comment_text(), it.short_label(), @@ -137,7 +140,8 @@ fn named_target(file_id: FileId, node: &SyntaxNode) -> Option }, ast::FnDef(it) => { Some(NavigationTarget::from_named( - file_id, + db, + file_id.into(), &it, it.doc_comment_text(), it.short_label(), @@ -145,7 +149,8 @@ fn named_target(file_id: FileId, node: &SyntaxNode) -> Option }, ast::TypeAliasDef(it) => { Some(NavigationTarget::from_named( - file_id, + db, + file_id.into(), &it, it.doc_comment_text(), it.short_label(), @@ -153,7 +158,8 @@ fn named_target(file_id: FileId, node: &SyntaxNode) -> Option }, ast::ConstDef(it) => { Some(NavigationTarget::from_named( - file_id, + db, + file_id.into(), &it, it.doc_comment_text(), it.short_label(), @@ -161,7 +167,8 @@ fn named_target(file_id: FileId, node: &SyntaxNode) -> Option }, ast::StaticDef(it) => { Some(NavigationTarget::from_named( - file_id, + db, + file_id.into(), &it, it.doc_comment_text(), it.short_label(), @@ -169,7 +176,8 @@ fn named_target(file_id: FileId, node: &SyntaxNode) -> Option }, ast::TraitDef(it) => { Some(NavigationTarget::from_named( - file_id, + db, + file_id.into(), &it, it.doc_comment_text(), it.short_label(), @@ -177,7 +185,8 @@ fn named_target(file_id: FileId, node: &SyntaxNode) -> Option }, ast::RecordFieldDef(it) => { Some(NavigationTarget::from_named( - file_id, + db, + file_id.into(), &it, it.doc_comment_text(), it.short_label(), @@ -185,7 +194,8 @@ fn named_target(file_id: FileId, node: &SyntaxNode) -> Option }, ast::Module(it) => { Some(NavigationTarget::from_named( - file_id, + db, + file_id.into(), &it, it.doc_comment_text(), it.short_label(), @@ -193,7 +203,8 @@ fn named_target(file_id: FileId, node: &SyntaxNode) -> Option }, ast::MacroCall(it) => { Some(NavigationTarget::from_named( - file_id, + db, + file_id.into(), &it, it.doc_comment_text(), None, From 1630a34c3f5c8b1ae2b4e1a99a918fd4d4ec52aa Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 4 Nov 2019 01:49:49 +0800 Subject: [PATCH 07/16] Add tests --- crates/ra_ide_api/src/goto_definition.rs | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/crates/ra_ide_api/src/goto_definition.rs b/crates/ra_ide_api/src/goto_definition.rs index 97127706f3..afa59cbe3a 100644 --- a/crates/ra_ide_api/src/goto_definition.rs +++ b/crates/ra_ide_api/src/goto_definition.rs @@ -345,6 +345,46 @@ mod tests { ); } + #[test] + fn goto_definition_works_for_macro_defined_fn_with_arg() { + check_goto( + " + //- /lib.rs + macro_rules! define_fn { + ($name:ident) => (fn $name() {}) + } + + define_fn!( + foo + ) + + fn bar() { + <|>foo(); + } + ", + "foo FN_DEF FileId(1) [80; 83) [80; 83)", + ); + } + + #[test] + fn goto_definition_works_for_macro_defined_fn_no_arg() { + check_goto( + " + //- /lib.rs + macro_rules! define_fn { + () => (fn foo() {}) + } + + define_fn!(); + + fn bar() { + <|>foo(); + } + ", + "foo FN_DEF FileId(1) [39; 42) [39; 42)", + ); + } + #[test] fn goto_definition_works_for_methods() { covers!(goto_definition_works_for_methods); From d8b7ba201e005e533d132f4942ad611468f21a28 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 4 Nov 2019 01:56:38 +0800 Subject: [PATCH 08/16] Add note for recurseive macro generated code --- crates/ra_ide_api/src/display/navigation_target.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/ra_ide_api/src/display/navigation_target.rs b/crates/ra_ide_api/src/display/navigation_target.rs index 0c3e25ce67..b9e3132f40 100644 --- a/crates/ra_ide_api/src/display/navigation_target.rs +++ b/crates/ra_ide_api/src/display/navigation_target.rs @@ -40,6 +40,7 @@ fn find_range_from_node( .and_then(|(files, expansion_info)| expansion_info.find_range(text_range, files)) .unwrap_or((src, text_range)); + // FIXME: handle recursive macro generated macro (file_id.original_file(db), text_range) } From e6709f64af1836460aee41aca34eb19ed3a337dd Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 4 Nov 2019 03:12:19 +0800 Subject: [PATCH 09/16] Rename and fix typos --- crates/ra_hir_expand/src/db.rs | 28 +++++++++++++++++++--------- crates/ra_hir_expand/src/lib.rs | 2 +- crates/ra_mbe/src/syntax_bridge.rs | 24 +++++++++++++++--------- crates/ra_tt/src/lib.rs | 13 ++++++++----- 4 files changed, 43 insertions(+), 24 deletions(-) diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs index 8abfbb4ffa..b3746924d2 100644 --- a/crates/ra_hir_expand/src/db.rs +++ b/crates/ra_hir_expand/src/db.rs @@ -18,6 +18,12 @@ pub struct ParseMacroWithInfo { pub expansion_info: Arc, } +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct MacroExpandInfo { + pub arg_map: Arc, + pub def_map: Arc, +} + // FIXME: rename to ExpandDatabase #[salsa::query_group(AstDatabaseStorage)] pub trait AstDatabase: SourceDatabase { @@ -35,7 +41,7 @@ pub trait AstDatabase: SourceDatabase { fn macro_expand( &self, macro_call: MacroCallId, - ) -> Result<(Arc, (Arc, Arc)), String>; + ) -> Result<(Arc, MacroExpandInfo), String>; fn macro_expansion_info(&self, macro_file: MacroFile) -> Option>; } @@ -77,7 +83,7 @@ pub(crate) fn macro_arg( pub(crate) fn macro_expand( db: &dyn AstDatabase, id: MacroCallId, -) -> Result<(Arc, (Arc, Arc)), String> { +) -> Result<(Arc, MacroExpandInfo), String> { let loc = db.lookup_intern_macro(id); let macro_arg = db.macro_arg(id).ok_or("Fail to args in to tt::TokenTree")?; @@ -89,7 +95,10 @@ pub(crate) fn macro_expand( return Err(format!("Total tokens count exceed limit : count = {}", count)); } - Ok((Arc::new(tt), (macro_arg.1.clone(), macro_rules.1.clone()))) + Ok(( + Arc::new(tt), + MacroExpandInfo { arg_map: macro_arg.1.clone(), def_map: macro_rules.1.clone() }, + )) } pub(crate) fn parse_or_expand(db: &dyn AstDatabase, file_id: HirFileId) -> Option { @@ -133,7 +142,7 @@ pub(crate) fn parse_macro_with_info( }; res.map(|(parsed, exp_map)| { - let (arg_map, def_map) = tt.1; + let expand_info = tt.1; let loc: MacroCallLoc = db.lookup_intern_macro(macro_call_id); let def_start = @@ -141,11 +150,12 @@ pub(crate) fn parse_macro_with_info( let arg_start = loc.ast_id.to_node(db).token_tree().map(|t| t.syntax().text_range().start()); - let arg_map = - arg_start.map(|start| exp_map.ranges(&arg_map, start)).unwrap_or_else(|| Vec::new()); - - let def_map = - def_start.map(|start| exp_map.ranges(&def_map, start)).unwrap_or_else(|| Vec::new()); + let arg_map = arg_start + .map(|start| exp_map.ranges(&expand_info.arg_map, start)) + .unwrap_or_else(|| Vec::new()); + let def_map = def_start + .map(|start| exp_map.ranges(&expand_info.def_map, start)) + .unwrap_or_else(|| Vec::new()); let info = ExpansionInfo { arg_map, def_map }; diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs index b6e55aa6ad..63ba17158c 100644 --- a/crates/ra_hir_expand/src/lib.rs +++ b/crates/ra_hir_expand/src/lib.rs @@ -132,7 +132,7 @@ impl MacroCallId { } #[derive(Debug, Clone, PartialEq, Eq)] -/// ExpansionInfo mainly describle how to map text range between src and expaned macro +/// ExpansionInfo mainly describes how to map text range between src and expanded macro pub struct ExpansionInfo { pub arg_map: Vec<(TextRange, TextRange)>, pub def_map: Vec<(TextRange, TextRange)>, diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 8e86742d03..2f21ad6e0a 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -26,7 +26,11 @@ static TOKEN_MAP_COUNTER: AtomicU32 = AtomicU32::new(0); /// Generate an unique token map id for each instance fn make_uniq_token_map_id() -> u32 { - TOKEN_MAP_COUNTER.fetch_add(1, Ordering::SeqCst) + let res = TOKEN_MAP_COUNTER.fetch_add(1, Ordering::SeqCst); + if res == std::u32::MAX { + panic!("TOKEN_MAP_COUNTER is overflowed"); + } + res } impl std::default::Default for TokenMap { @@ -35,10 +39,9 @@ impl std::default::Default for TokenMap { } } -/// Maps Relative range of the expanded syntax node to `tt::TokenId` +/// Maps relative range of the expanded syntax node to `tt::TokenId` #[derive(Debug, PartialEq, Eq, Default)] pub struct ExpandedRangeMap { - /// Maps `tt::TokenId` to the *relative* source range. ranges: Vec<(TextRange, tt::TokenId)>, } @@ -85,14 +88,13 @@ fn fragment_to_syntax_node( }; let buffer = TokenBuffer::new(&tokens); let mut token_source = SubtreeTokenSource::new(&buffer); - let mut range_map = ExpandedRangeMap::default(); - let mut tree_sink = TtTreeSink::new(buffer.begin(), &mut range_map); + let mut tree_sink = TtTreeSink::new(buffer.begin()); ra_parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } //FIXME: would be cool to report errors - let parse = tree_sink.inner.finish(); + let (parse, range_map) = tree_sink.finish(); Ok((parse, range_map)) } @@ -320,7 +322,7 @@ struct TtTreeSink<'a> { cursor: Cursor<'a>, text_pos: TextUnit, inner: SyntaxTreeBuilder, - range_map: &'a mut ExpandedRangeMap, + range_map: ExpandedRangeMap, // Number of roots // Use for detect ill-form tree which is not single root @@ -328,16 +330,20 @@ struct TtTreeSink<'a> { } impl<'a> TtTreeSink<'a> { - fn new(cursor: Cursor<'a>, range_map: &'a mut ExpandedRangeMap) -> Self { + fn new(cursor: Cursor<'a>) -> Self { TtTreeSink { buf: String::new(), cursor, text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), roots: smallvec::SmallVec::new(), - range_map, + range_map: ExpandedRangeMap::default(), } } + + fn finish(self) -> (Parse, ExpandedRangeMap) { + (self.inner.finish(), self.range_map) + } } fn delim_to_str(d: tt::Delimiter, closing: bool) -> SmolStr { diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs index 96410ff222..1449618f3e 100644 --- a/crates/ra_tt/src/lib.rs +++ b/crates/ra_tt/src/lib.rs @@ -25,23 +25,26 @@ use smol_str::SmolStr; /// source token and making sure that identities are preserved during macro /// expansion. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct TokenId(u32, u32); +pub struct TokenId { + token_id: u32, + map_id: u32, +} impl TokenId { pub fn new(token_id: u32, map_id: u32) -> TokenId { - TokenId(token_id, map_id) + TokenId { token_id, map_id } } pub const fn unspecified() -> TokenId { - TokenId(!0, !0) + TokenId { token_id: !0, map_id: !0 } } pub fn token_id(&self) -> u32 { - self.0 + self.token_id } pub fn map_id(&self) -> u32 { - self.1 + self.map_id } } From 7e28924012409352bbacbfebf9fac4e4409f09b8 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 4 Nov 2019 21:43:36 +0800 Subject: [PATCH 10/16] Use ? and destructing to simplifed long code --- crates/ra_hir_expand/src/db.rs | 36 ++++++++++++++++------------------ 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs index b3746924d2..8369c2b40f 100644 --- a/crates/ra_hir_expand/src/db.rs +++ b/crates/ra_hir_expand/src/db.rs @@ -132,35 +132,33 @@ pub(crate) fn parse_macro_with_info( log::warn!("fail on macro_parse: (reason: {})", err,); }) .ok()?; - let res = match macro_file.macro_file_kind { + + let (parsed, exp_map) = match macro_file.macro_file_kind { MacroFileKind::Items => { - mbe::token_tree_to_items(&tt.0).ok().map(|(p, map)| (Parse::to_syntax(p), map)) + mbe::token_tree_to_items(&tt.0).map(|(p, map)| (p.to_syntax(), map)).ok()? } MacroFileKind::Expr => { - mbe::token_tree_to_expr(&tt.0).ok().map(|(p, map)| (Parse::to_syntax(p), map)) + mbe::token_tree_to_expr(&tt.0).map(|(p, map)| (p.to_syntax(), map)).ok()? } }; - res.map(|(parsed, exp_map)| { - let expand_info = tt.1; - let loc: MacroCallLoc = db.lookup_intern_macro(macro_call_id); + let expand_info = tt.1; + let loc: MacroCallLoc = db.lookup_intern_macro(macro_call_id); - let def_start = - loc.def.ast_id.to_node(db).token_tree().map(|t| t.syntax().text_range().start()); - let arg_start = - loc.ast_id.to_node(db).token_tree().map(|t| t.syntax().text_range().start()); + let arg_tt = loc.ast_id.to_node(db).token_tree(); + let def_tt = loc.def.ast_id.to_node(db).token_tree(); - let arg_map = arg_start - .map(|start| exp_map.ranges(&expand_info.arg_map, start)) - .unwrap_or_else(|| Vec::new()); - let def_map = def_start - .map(|start| exp_map.ranges(&expand_info.def_map, start)) - .unwrap_or_else(|| Vec::new()); + let arg_start = arg_tt.map(|t| t.syntax().text_range().start()); + let def_start = def_tt.map(|t| t.syntax().text_range().start()); - let info = ExpansionInfo { arg_map, def_map }; + let arg_map = + arg_start.map(|start| exp_map.ranges(&expand_info.arg_map, start)).unwrap_or_default(); + let def_map = + def_start.map(|start| exp_map.ranges(&expand_info.def_map, start)).unwrap_or_default(); - ParseMacroWithInfo { parsed, expansion_info: Arc::new(info) } - }) + let info = ExpansionInfo { arg_map, def_map }; + + Some(ParseMacroWithInfo { parsed, expansion_info: Arc::new(info) }) } pub(crate) fn macro_expansion_info( From 604bdc6ffeea69dffed6d03f10b2fd4e05d04cda Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Tue, 5 Nov 2019 02:09:16 +0800 Subject: [PATCH 11/16] Use macro_rules shift to map text ranges --- crates/ra_hir_expand/src/db.rs | 13 +++++++--- crates/ra_mbe/src/lib.rs | 4 +++ crates/ra_mbe/src/syntax_bridge.rs | 41 ++++++++---------------------- crates/ra_tt/src/lib.rs | 19 ++------------ 4 files changed, 26 insertions(+), 51 deletions(-) diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs index 8369c2b40f..72c728c938 100644 --- a/crates/ra_hir_expand/src/db.rs +++ b/crates/ra_hir_expand/src/db.rs @@ -151,10 +151,15 @@ pub(crate) fn parse_macro_with_info( let arg_start = arg_tt.map(|t| t.syntax().text_range().start()); let def_start = def_tt.map(|t| t.syntax().text_range().start()); - let arg_map = - arg_start.map(|start| exp_map.ranges(&expand_info.arg_map, start)).unwrap_or_default(); - let def_map = - def_start.map(|start| exp_map.ranges(&expand_info.def_map, start)).unwrap_or_default(); + let macro_rules = db.macro_def(loc.def).ok_or("Fail to find macro definition").ok()?.0; + let shift = macro_rules.shift(); + + let arg_map = arg_start + .map(|start| exp_map.map_ranges(&expand_info.arg_map, start, shift)) + .unwrap_or_default(); + let def_map = def_start + .map(|start| exp_map.map_ranges(&expand_info.def_map, start, 0)) + .unwrap_or_default(); let info = ExpansionInfo { arg_map, def_map }; diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 2926b29fd0..a0f1a68813 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -118,6 +118,10 @@ impl MacroRules { shift_subtree(&mut tt, self.shift); mbe_expander::expand(self, &tt) } + + pub fn shift(&self) -> u32 { + self.shift + } } impl Rule { diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 2f21ad6e0a..0c499cd03c 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -12,31 +12,12 @@ use tt::buffer::{Cursor, TokenBuffer}; use crate::subtree_source::SubtreeTokenSource; use crate::ExpandError; -use std::sync::atomic::{AtomicU32, Ordering}; /// Maps `tt::TokenId` to the relative range of the original token. -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Default)] pub struct TokenMap { /// Maps `tt::TokenId` to the *relative* source range. tokens: Vec, - map_id: u32, -} - -static TOKEN_MAP_COUNTER: AtomicU32 = AtomicU32::new(0); - -/// Generate an unique token map id for each instance -fn make_uniq_token_map_id() -> u32 { - let res = TOKEN_MAP_COUNTER.fetch_add(1, Ordering::SeqCst); - if res == std::u32::MAX { - panic!("TOKEN_MAP_COUNTER is overflowed"); - } - res -} - -impl std::default::Default for TokenMap { - fn default() -> TokenMap { - TokenMap { tokens: Default::default(), map_id: make_uniq_token_map_id() } - } } /// Maps relative range of the expanded syntax node to `tt::TokenId` @@ -140,17 +121,14 @@ pub fn token_tree_to_items( impl TokenMap { pub fn relative_range_of(&self, tt: tt::TokenId) -> Option { - if self.map_id != tt.map_id() { - return None; - } - let idx = tt.token_id() as usize; + let idx = tt.0 as usize; self.tokens.get(idx).copied() } fn alloc(&mut self, relative_range: TextRange) -> tt::TokenId { let id = self.tokens.len(); self.tokens.push(relative_range); - tt::TokenId::new(id as u32, self.map_id) + tt::TokenId(id as u32) } } @@ -159,14 +137,17 @@ impl ExpandedRangeMap { self.ranges.push((relative_range, token_id.clone())) } - pub fn ranges(&self, to: &TokenMap, start: TextUnit) -> Vec<(TextRange, TextRange)> { + pub fn map_ranges( + &self, + to: &TokenMap, + start: TextUnit, + shift: u32, + ) -> Vec<(TextRange, TextRange)> { self.ranges .iter() + .filter_map(|(r, tid)| if shift <= tid.0 { Some((r, tid.0 - shift)) } else { None }) .filter_map(|(r, tid)| { - if to.map_id != tid.map_id() { - return None; - } - if let Some(to_range) = to.relative_range_of(*tid) { + if let Some(to_range) = to.relative_range_of(tt::TokenId(tid)) { Some((*r, TextRange::from_to(to_range.start() + start, to_range.end() + start))) } else { None diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs index 1449618f3e..20c251ff45 100644 --- a/crates/ra_tt/src/lib.rs +++ b/crates/ra_tt/src/lib.rs @@ -25,26 +25,11 @@ use smol_str::SmolStr; /// source token and making sure that identities are preserved during macro /// expansion. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct TokenId { - token_id: u32, - map_id: u32, -} +pub struct TokenId(pub u32); impl TokenId { - pub fn new(token_id: u32, map_id: u32) -> TokenId { - TokenId { token_id, map_id } - } - pub const fn unspecified() -> TokenId { - TokenId { token_id: !0, map_id: !0 } - } - - pub fn token_id(&self) -> u32 { - self.token_id - } - - pub fn map_id(&self) -> u32 { - self.map_id + TokenId(!0) } } From 2a76bb5a0b70ae83824aa2e094ae2f592dcee694 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Tue, 5 Nov 2019 03:20:54 +0800 Subject: [PATCH 12/16] Refactor a bit --- crates/ra_hir_expand/src/db.rs | 3 +-- crates/ra_mbe/src/syntax_bridge.rs | 15 ++++++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs index 72c728c938..e2860371f7 100644 --- a/crates/ra_hir_expand/src/db.rs +++ b/crates/ra_hir_expand/src/db.rs @@ -151,8 +151,7 @@ pub(crate) fn parse_macro_with_info( let arg_start = arg_tt.map(|t| t.syntax().text_range().start()); let def_start = def_tt.map(|t| t.syntax().text_range().start()); - let macro_rules = db.macro_def(loc.def).ok_or("Fail to find macro definition").ok()?.0; - let shift = macro_rules.shift(); + let shift = db.macro_def(loc.def)?.0.shift(); let arg_map = arg_start .map(|start| exp_map.map_ranges(&expand_info.arg_map, start, shift)) diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 0c499cd03c..02ddd25e7a 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -133,7 +133,7 @@ impl TokenMap { } impl ExpandedRangeMap { - fn set(&mut self, relative_range: TextRange, token_id: &tt::TokenId) { + fn add(&mut self, relative_range: TextRange, token_id: &tt::TokenId) { self.ranges.push((relative_range, token_id.clone())) } @@ -145,13 +145,11 @@ impl ExpandedRangeMap { ) -> Vec<(TextRange, TextRange)> { self.ranges .iter() - .filter_map(|(r, tid)| if shift <= tid.0 { Some((r, tid.0 - shift)) } else { None }) .filter_map(|(r, tid)| { - if let Some(to_range) = to.relative_range_of(tt::TokenId(tid)) { - Some((*r, TextRange::from_to(to_range.start() + start, to_range.end() + start))) - } else { - None - } + let adjusted_id = tt::TokenId(tid.0.checked_sub(shift)?); + let to_range = to.relative_range_of(adjusted_id)?; + + Some((*r, TextRange::offset_len(to_range.start() + start, to_range.len()))) }) .collect() } @@ -361,8 +359,7 @@ impl<'a> TreeSink for TtTreeSink<'a> { self.text_pos + TextUnit::of_str(&self.buf), TextUnit::of_str(&ident.text), ); - let token_id = ident.id; - self.range_map.set(range, &token_id); + self.range_map.add(range, &ident.id); } } From 568f3cff41fd4ef49468b60f4343755a5b7b05cb Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Tue, 5 Nov 2019 09:43:59 +0800 Subject: [PATCH 13/16] Fix formatting --- crates/ra_mbe/src/syntax_bridge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 02ddd25e7a..d897ac5de1 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -148,7 +148,7 @@ impl ExpandedRangeMap { .filter_map(|(r, tid)| { let adjusted_id = tt::TokenId(tid.0.checked_sub(shift)?); let to_range = to.relative_range_of(adjusted_id)?; - + Some((*r, TextRange::offset_len(to_range.start() + start, to_range.len()))) }) .collect() From 62ed93db13877888379ea38c843729419bdf99d0 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 8 Nov 2019 09:57:01 +0800 Subject: [PATCH 14/16] Refactor and add more comments --- crates/ra_hir_expand/src/db.rs | 14 ++--- crates/ra_mbe/src/syntax_bridge.rs | 96 ++++++++++++++---------------- 2 files changed, 52 insertions(+), 58 deletions(-) diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs index e2860371f7..1eaf8bf309 100644 --- a/crates/ra_hir_expand/src/db.rs +++ b/crates/ra_hir_expand/src/db.rs @@ -148,17 +148,15 @@ pub(crate) fn parse_macro_with_info( let arg_tt = loc.ast_id.to_node(db).token_tree(); let def_tt = loc.def.ast_id.to_node(db).token_tree(); - let arg_start = arg_tt.map(|t| t.syntax().text_range().start()); - let def_start = def_tt.map(|t| t.syntax().text_range().start()); + let arg_range = arg_tt.map(|t| t.syntax().text_range()); + let def_range = def_tt.map(|t| t.syntax().text_range()); let shift = db.macro_def(loc.def)?.0.shift(); - let arg_map = arg_start - .map(|start| exp_map.map_ranges(&expand_info.arg_map, start, shift)) - .unwrap_or_default(); - let def_map = def_start - .map(|start| exp_map.map_ranges(&expand_info.def_map, start, 0)) - .unwrap_or_default(); + let arg_map = + arg_range.map(|it| exp_map.map_ranges(&expand_info.arg_map, it, shift)).unwrap_or_default(); + let def_map = + def_range.map(|it| exp_map.map_ranges(&expand_info.def_map, it, 0)).unwrap_or_default(); let info = ExpansionInfo { arg_map, def_map }; diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index d897ac5de1..a49e63ace4 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -22,7 +22,7 @@ pub struct TokenMap { /// Maps relative range of the expanded syntax node to `tt::TokenId` #[derive(Debug, PartialEq, Eq, Default)] -pub struct ExpandedRangeMap { +pub struct RevTokenMap { ranges: Vec<(TextRange, tt::TokenId)>, } @@ -58,7 +58,7 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke fn fragment_to_syntax_node( tt: &tt::Subtree, fragment_kind: FragmentKind, -) -> Result<(Parse, ExpandedRangeMap), ExpandError> { +) -> Result<(Parse, RevTokenMap), ExpandError> { let tmp; let tokens = match tt { tt::Subtree { delimiter: tt::Delimiter::None, token_trees } => token_trees.as_slice(), @@ -79,44 +79,29 @@ fn fragment_to_syntax_node( Ok((parse, range_map)) } -/// Parses the token tree (result of macro expansion) to an expression -pub fn token_tree_to_expr( - tt: &tt::Subtree, -) -> Result<(Parse, ExpandedRangeMap), ExpandError> { - let (parse, map) = fragment_to_syntax_node(tt, Expr)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) +macro_rules! impl_token_tree_conversions { + ($($(#[$attr:meta])* $name:ident => ($kind:ident, $t:ty) ),*) => { + $( + $(#[$attr])* + pub fn $name(tt: &tt::Subtree) -> Result<(Parse<$t>, RevTokenMap), ExpandError> { + let (parse, map) = fragment_to_syntax_node(tt, $kind)?; + parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) + } + )* + } } -/// Parses the token tree (result of macro expansion) to a Pattern -pub fn token_tree_to_pat( - tt: &tt::Subtree, -) -> Result<(Parse, ExpandedRangeMap), ExpandError> { - let (parse, map) = fragment_to_syntax_node(tt, Pattern)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) -} - -/// Parses the token tree (result of macro expansion) to a Type -pub fn token_tree_to_ty( - tt: &tt::Subtree, -) -> Result<(Parse, ExpandedRangeMap), ExpandError> { - let (parse, map) = fragment_to_syntax_node(tt, Type)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) -} - -/// Parses the token tree (result of macro expansion) as a sequence of stmts -pub fn token_tree_to_macro_stmts( - tt: &tt::Subtree, -) -> Result<(Parse, ExpandedRangeMap), ExpandError> { - let (parse, map) = fragment_to_syntax_node(tt, Statements)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) -} - -/// Parses the token tree (result of macro expansion) as a sequence of items -pub fn token_tree_to_items( - tt: &tt::Subtree, -) -> Result<(Parse, ExpandedRangeMap), ExpandError> { - let (parse, map) = fragment_to_syntax_node(tt, Items)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) +impl_token_tree_conversions! { + /// Parses the token tree (result of macro expansion) to an expression + token_tree_to_expr => (Expr, ast::Expr), + /// Parses the token tree (result of macro expansion) to a Pattern + token_tree_to_pat => (Pattern, ast::Pat), + /// Parses the token tree (result of macro expansion) to a Type + token_tree_to_ty => (Type, ast::TypeRef), + /// Parses the token tree (result of macro expansion) as a sequence of stmts + token_tree_to_macro_stmts => (Statements, ast::MacroStmts), + /// Parses the token tree (result of macro expansion) as a sequence of items + token_tree_to_items => (Items, ast::MacroItems) } impl TokenMap { @@ -132,15 +117,28 @@ impl TokenMap { } } -impl ExpandedRangeMap { - fn add(&mut self, relative_range: TextRange, token_id: &tt::TokenId) { +impl RevTokenMap { + fn add(&mut self, relative_range: TextRange, token_id: tt::TokenId) { self.ranges.push((relative_range, token_id.clone())) } + /// Map a given token map to (Expanded syntax node, Input tokens) text-ranges pair + /// + /// This function do the following things: + /// + /// 1. Undo the increment of token-id `shift`: + /// When we output a token from from macro argument, we increased its id + /// by `shift` (so it's guaranteed to not to collide with anything from the definition) + /// We undo the increment here to rollback to its original token id. + /// 2. Offset the input tokens (`to`) by `parent` text-range: + /// We transforms the input tokens text-ranges from relative to original first token + /// to parent text-range + /// 3. Maps expanded tokens text-ranges to parent text-ranges + /// pub fn map_ranges( &self, to: &TokenMap, - start: TextUnit, + parent: TextRange, shift: u32, ) -> Vec<(TextRange, TextRange)> { self.ranges @@ -149,7 +147,7 @@ impl ExpandedRangeMap { let adjusted_id = tt::TokenId(tid.0.checked_sub(shift)?); let to_range = to.relative_range_of(adjusted_id)?; - Some((*r, TextRange::offset_len(to_range.start() + start, to_range.len()))) + Some((*r, TextRange::offset_len(to_range.start() + parent.start(), to_range.len()))) }) .collect() } @@ -301,7 +299,7 @@ struct TtTreeSink<'a> { cursor: Cursor<'a>, text_pos: TextUnit, inner: SyntaxTreeBuilder, - range_map: ExpandedRangeMap, + range_map: RevTokenMap, // Number of roots // Use for detect ill-form tree which is not single root @@ -316,11 +314,11 @@ impl<'a> TtTreeSink<'a> { text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), roots: smallvec::SmallVec::new(), - range_map: ExpandedRangeMap::default(), + range_map: RevTokenMap::default(), } } - fn finish(self) -> (Parse, ExpandedRangeMap) { + fn finish(self) -> (Parse, RevTokenMap) { (self.inner.finish(), self.range_map) } } @@ -355,11 +353,9 @@ impl<'a> TreeSink for TtTreeSink<'a> { // Mark the range if needed if let tt::Leaf::Ident(ident) = leaf { if kind == IDENT { - let range = TextRange::offset_len( - self.text_pos + TextUnit::of_str(&self.buf), - TextUnit::of_str(&ident.text), - ); - self.range_map.add(range, &ident.id); + let range = + TextRange::offset_len(self.text_pos, TextUnit::of_str(&ident.text)); + self.range_map.add(range, ident.id); } } From d01e0abdb57c5be340934cb51e2f6b1a2e6c0373 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 9 Nov 2019 04:00:27 +0800 Subject: [PATCH 15/16] Refactor and simpfily --- crates/ra_hir_expand/src/db.rs | 80 ++++--------------- crates/ra_hir_expand/src/lib.rs | 42 ++++++---- .../src/display/navigation_target.rs | 5 +- crates/ra_ide_api/src/status.rs | 4 +- crates/ra_mbe/src/lib.rs | 2 +- 5 files changed, 43 insertions(+), 90 deletions(-) diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs index 1eaf8bf309..e6d2e8d9dc 100644 --- a/crates/ra_hir_expand/src/db.rs +++ b/crates/ra_hir_expand/src/db.rs @@ -8,22 +8,10 @@ use ra_prof::profile; use ra_syntax::{AstNode, Parse, SyntaxNode}; use crate::{ - ast_id_map::AstIdMap, ExpansionInfo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallLoc, - MacroDefId, MacroFile, MacroFileKind, + ast_id_map::AstIdMap, HirFileId, HirFileIdRepr, MacroCallId, MacroCallLoc, MacroDefId, + MacroFile, MacroFileKind, }; -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct ParseMacroWithInfo { - pub parsed: Parse, - pub expansion_info: Arc, -} - -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct MacroExpandInfo { - pub arg_map: Arc, - pub def_map: Arc, -} - // FIXME: rename to ExpandDatabase #[salsa::query_group(AstDatabaseStorage)] pub trait AstDatabase: SourceDatabase { @@ -36,14 +24,11 @@ pub trait AstDatabase: SourceDatabase { fn intern_macro(&self, macro_call: MacroCallLoc) -> MacroCallId; fn macro_arg(&self, id: MacroCallId) -> Option<(Arc, Arc)>; fn macro_def(&self, id: MacroDefId) -> Option<(Arc, Arc)>; - fn parse_macro(&self, macro_file: MacroFile) -> Option>; - fn parse_macro_with_info(&self, macro_file: MacroFile) -> Option; - fn macro_expand( + fn parse_macro( &self, - macro_call: MacroCallId, - ) -> Result<(Arc, MacroExpandInfo), String>; - - fn macro_expansion_info(&self, macro_file: MacroFile) -> Option>; + macro_file: MacroFile, + ) -> Option<(Parse, Arc)>; + fn macro_expand(&self, macro_call: MacroCallId) -> Result, String>; } pub(crate) fn ast_id_map(db: &dyn AstDatabase, file_id: HirFileId) -> Arc { @@ -83,7 +68,7 @@ pub(crate) fn macro_arg( pub(crate) fn macro_expand( db: &dyn AstDatabase, id: MacroCallId, -) -> Result<(Arc, MacroExpandInfo), String> { +) -> Result, String> { let loc = db.lookup_intern_macro(id); let macro_arg = db.macro_arg(id).ok_or("Fail to args in to tt::TokenTree")?; @@ -94,18 +79,14 @@ pub(crate) fn macro_expand( if count > 65536 { return Err(format!("Total tokens count exceed limit : count = {}", count)); } - - Ok(( - Arc::new(tt), - MacroExpandInfo { arg_map: macro_arg.1.clone(), def_map: macro_rules.1.clone() }, - )) + Ok(Arc::new(tt)) } pub(crate) fn parse_or_expand(db: &dyn AstDatabase, file_id: HirFileId) -> Option { match file_id.0 { HirFileIdRepr::FileId(file_id) => Some(db.parse(file_id).tree().syntax().clone()), HirFileIdRepr::MacroFile(macro_file) => { - db.parse_macro(macro_file).map(|it| it.syntax_node()) + db.parse_macro(macro_file).map(|(it, _)| it.syntax_node()) } } } @@ -113,15 +94,9 @@ pub(crate) fn parse_or_expand(db: &dyn AstDatabase, file_id: HirFileId) -> Optio pub(crate) fn parse_macro( db: &dyn AstDatabase, macro_file: MacroFile, -) -> Option> { +) -> Option<(Parse, Arc)> { let _p = profile("parse_macro_query"); - db.parse_macro_with_info(macro_file).map(|r| r.parsed) -} -pub(crate) fn parse_macro_with_info( - db: &dyn AstDatabase, - macro_file: MacroFile, -) -> Option { let macro_call_id = macro_file.macro_call_id; let tt = db .macro_expand(macro_call_id) @@ -133,39 +108,12 @@ pub(crate) fn parse_macro_with_info( }) .ok()?; - let (parsed, exp_map) = match macro_file.macro_file_kind { + match macro_file.macro_file_kind { MacroFileKind::Items => { - mbe::token_tree_to_items(&tt.0).map(|(p, map)| (p.to_syntax(), map)).ok()? + mbe::token_tree_to_items(&tt).ok().map(|(p, map)| (p.to_syntax(), Arc::new(map))) } MacroFileKind::Expr => { - mbe::token_tree_to_expr(&tt.0).map(|(p, map)| (p.to_syntax(), map)).ok()? + mbe::token_tree_to_expr(&tt).ok().map(|(p, map)| (p.to_syntax(), Arc::new(map))) } - }; - - let expand_info = tt.1; - let loc: MacroCallLoc = db.lookup_intern_macro(macro_call_id); - - let arg_tt = loc.ast_id.to_node(db).token_tree(); - let def_tt = loc.def.ast_id.to_node(db).token_tree(); - - let arg_range = arg_tt.map(|t| t.syntax().text_range()); - let def_range = def_tt.map(|t| t.syntax().text_range()); - - let shift = db.macro_def(loc.def)?.0.shift(); - - let arg_map = - arg_range.map(|it| exp_map.map_ranges(&expand_info.arg_map, it, shift)).unwrap_or_default(); - let def_map = - def_range.map(|it| exp_map.map_ranges(&expand_info.def_map, it, 0)).unwrap_or_default(); - - let info = ExpansionInfo { arg_map, def_map }; - - Some(ParseMacroWithInfo { parsed, expansion_info: Arc::new(info) }) -} - -pub(crate) fn macro_expansion_info( - db: &dyn AstDatabase, - macro_file: MacroFile, -) -> Option> { - db.parse_macro_with_info(macro_file).map(|res| res.expansion_info.clone()) + } } diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs index 63ba17158c..1a1d6bdd42 100644 --- a/crates/ra_hir_expand/src/lib.rs +++ b/crates/ra_hir_expand/src/lib.rs @@ -20,7 +20,6 @@ use ra_syntax::{ }; use crate::ast_id_map::FileAstId; -use std::sync::Arc; /// Input to the analyzer is a set of files, where each file is identified by /// `FileId` and contains source code. However, another source of source code in @@ -69,19 +68,29 @@ impl HirFileId { } /// Return expansion information if it is a macro-expansion file - pub fn parent_expansion( - self, - db: &dyn db::AstDatabase, - ) -> Option<((HirFileId, HirFileId), Arc)> { + pub fn parent_expansion(self, db: &dyn db::AstDatabase) -> Option { match self.0 { HirFileIdRepr::FileId(_) => None, HirFileIdRepr::MacroFile(macro_file) => { let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id); - let def_file = loc.def.ast_id.file_id; - let arg_file = loc.ast_id.file_id; + let arg_range = loc.ast_id.to_node(db).token_tree()?.syntax().text_range(); + let def_range = loc.def.ast_id.to_node(db).token_tree()?.syntax().text_range(); - db.macro_expansion_info(macro_file).map(|ex| ((arg_file, def_file), ex)) + let macro_def = db.macro_def(loc.def)?; + let shift = macro_def.0.shift(); + let rev_map = db.parse_macro(macro_file)?.1; + + let arg_token_map = db.macro_arg(macro_file.macro_call_id)?.1; + let def_token_map = macro_def.1; + + let arg_map = rev_map.map_ranges(&arg_token_map, arg_range, shift); + let def_map = rev_map.map_ranges(&def_token_map, def_range, 0); + + let arg_file = loc.ast_id.file_id; + let def_file = loc.def.ast_id.file_id; + + Some(ExpansionInfo { arg_file, def_file, arg_map, def_map }) } } } @@ -134,25 +143,24 @@ impl MacroCallId { #[derive(Debug, Clone, PartialEq, Eq)] /// ExpansionInfo mainly describes how to map text range between src and expanded macro pub struct ExpansionInfo { - pub arg_map: Vec<(TextRange, TextRange)>, - pub def_map: Vec<(TextRange, TextRange)>, + pub(crate) arg_file: HirFileId, + pub(crate) def_file: HirFileId, + + pub(crate) arg_map: Vec<(TextRange, TextRange)>, + pub(crate) def_map: Vec<(TextRange, TextRange)>, } impl ExpansionInfo { - pub fn find_range( - &self, - from: TextRange, - (arg_file_id, def_file_id): (HirFileId, HirFileId), - ) -> Option<(HirFileId, TextRange)> { + pub fn find_range(&self, from: TextRange) -> Option<(HirFileId, TextRange)> { for (src, dest) in &self.arg_map { if src.is_subrange(&from) { - return Some((arg_file_id, *dest)); + return Some((self.arg_file, *dest)); } } for (src, dest) in &self.def_map { if src.is_subrange(&from) { - return Some((def_file_id, *dest)); + return Some((self.def_file, *dest)); } } diff --git a/crates/ra_ide_api/src/display/navigation_target.rs b/crates/ra_ide_api/src/display/navigation_target.rs index b9e3132f40..b77e192313 100644 --- a/crates/ra_ide_api/src/display/navigation_target.rs +++ b/crates/ra_ide_api/src/display/navigation_target.rs @@ -37,7 +37,7 @@ fn find_range_from_node( let text_range = node.text_range(); let (file_id, text_range) = src .parent_expansion(db) - .and_then(|(files, expansion_info)| expansion_info.find_range(text_range, files)) + .and_then(|expansion_info| expansion_info.find_range(text_range)) .unwrap_or((src, text_range)); // FIXME: handle recursive macro generated macro @@ -139,7 +139,6 @@ impl NavigationTarget { pub(crate) fn from_module(db: &RootDatabase, module: hir::Module) -> NavigationTarget { let src = module.definition_source(db); let name = module.name(db).map(|it| it.to_string().into()).unwrap_or_default(); - match src.ast { ModuleSource::SourceFile(node) => { let (file_id, text_range) = find_range_from_node(db, src.file_id, node.syntax()); @@ -324,9 +323,7 @@ impl NavigationTarget { ) -> NavigationTarget { //FIXME: use `_` instead of empty string let name = node.name().map(|it| it.text().clone()).unwrap_or_default(); - let focus_range = node.name().map(|it| find_range_from_node(db, file_id, it.syntax()).1); - let (file_id, full_range) = find_range_from_node(db, file_id, node.syntax()); NavigationTarget::from_syntax( diff --git a/crates/ra_ide_api/src/status.rs b/crates/ra_ide_api/src/status.rs index f91f16c8e4..1bb27eb851 100644 --- a/crates/ra_ide_api/src/status.rs +++ b/crates/ra_ide_api/src/status.rs @@ -94,10 +94,10 @@ impl FromIterator>> for SyntaxTreeStat } } -impl FromIterator>>> for SyntaxTreeStats { +impl FromIterator, M)>>> for SyntaxTreeStats { fn from_iter(iter: T) -> SyntaxTreeStats where - T: IntoIterator>>>, + T: IntoIterator, M)>>>, { let mut res = SyntaxTreeStats::default(); for entry in iter { diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index a0f1a68813..70a289f099 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -32,7 +32,7 @@ pub enum ExpandError { pub use crate::syntax_bridge::{ ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_expr, token_tree_to_items, - token_tree_to_macro_stmts, token_tree_to_pat, token_tree_to_ty, TokenMap, + token_tree_to_macro_stmts, token_tree_to_pat, token_tree_to_ty, RevTokenMap, TokenMap, }; /// This struct contains AST for a single `macro_rules` definition. What might From 0a5ec69404a2556dd82e5bb00b295aebaa291f04 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 9 Nov 2019 12:00:46 +0800 Subject: [PATCH 16/16] Remove map_ranges in RevTokenMap --- crates/ra_hir_expand/src/db.rs | 12 ++-- crates/ra_hir_expand/src/lib.rs | 55 +++++++++---------- .../src/display/navigation_target.rs | 2 +- crates/ra_mbe/src/syntax_bridge.rs | 32 +---------- 4 files changed, 35 insertions(+), 66 deletions(-) diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs index e6d2e8d9dc..b789c6e7b3 100644 --- a/crates/ra_hir_expand/src/db.rs +++ b/crates/ra_hir_expand/src/db.rs @@ -22,8 +22,8 @@ pub trait AstDatabase: SourceDatabase { #[salsa::interned] fn intern_macro(&self, macro_call: MacroCallLoc) -> MacroCallId; - fn macro_arg(&self, id: MacroCallId) -> Option<(Arc, Arc)>; - fn macro_def(&self, id: MacroDefId) -> Option<(Arc, Arc)>; + fn macro_arg(&self, id: MacroCallId) -> Option>; + fn macro_def(&self, id: MacroDefId) -> Option>; fn parse_macro( &self, macro_file: MacroFile, @@ -40,7 +40,7 @@ pub(crate) fn ast_id_map(db: &dyn AstDatabase, file_id: HirFileId) -> Arc Option<(Arc, Arc)> { +) -> Option> { let macro_call = id.ast_id.to_node(db); let arg = macro_call.token_tree()?; let (tt, tmap) = mbe::ast_to_token_tree(&arg).or_else(|| { @@ -51,18 +51,18 @@ pub(crate) fn macro_def( log::warn!("fail on macro_def parse: {:#?}", tt); None })?; - Some((Arc::new(rules), Arc::new(tmap))) + Some(Arc::new((rules, tmap))) } pub(crate) fn macro_arg( db: &dyn AstDatabase, id: MacroCallId, -) -> Option<(Arc, Arc)> { +) -> Option> { let loc = db.lookup_intern_macro(id); let macro_call = loc.ast_id.to_node(db); let arg = macro_call.token_tree()?; let (tt, tmap) = mbe::ast_to_token_tree(&arg)?; - Some((Arc::new(tt), Arc::new(tmap))) + Some(Arc::new((tt, tmap))) } pub(crate) fn macro_expand( diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs index 1a1d6bdd42..b219b8fbf8 100644 --- a/crates/ra_hir_expand/src/lib.rs +++ b/crates/ra_hir_expand/src/lib.rs @@ -12,11 +12,12 @@ pub mod hygiene; pub mod diagnostics; use std::hash::{Hash, Hasher}; +use std::sync::Arc; use ra_db::{salsa, CrateId, FileId}; use ra_syntax::{ ast::{self, AstNode}, - SyntaxNode, TextRange, + SyntaxNode, TextRange, TextUnit, }; use crate::ast_id_map::FileAstId; @@ -68,29 +69,25 @@ impl HirFileId { } /// Return expansion information if it is a macro-expansion file - pub fn parent_expansion(self, db: &dyn db::AstDatabase) -> Option { + pub fn expansion_info(self, db: &dyn db::AstDatabase) -> Option { match self.0 { HirFileIdRepr::FileId(_) => None, HirFileIdRepr::MacroFile(macro_file) => { let loc: MacroCallLoc = db.lookup_intern_macro(macro_file.macro_call_id); - let arg_range = loc.ast_id.to_node(db).token_tree()?.syntax().text_range(); - let def_range = loc.def.ast_id.to_node(db).token_tree()?.syntax().text_range(); + let arg_start = loc.ast_id.to_node(db).token_tree()?.syntax().text_range().start(); + let def_start = + loc.def.ast_id.to_node(db).token_tree()?.syntax().text_range().start(); let macro_def = db.macro_def(loc.def)?; let shift = macro_def.0.shift(); - let rev_map = db.parse_macro(macro_file)?.1; + let exp_map = db.parse_macro(macro_file)?.1; + let macro_arg = db.macro_arg(macro_file.macro_call_id)?; - let arg_token_map = db.macro_arg(macro_file.macro_call_id)?.1; - let def_token_map = macro_def.1; + let arg_start = (loc.ast_id.file_id, arg_start); + let def_start = (loc.def.ast_id.file_id, def_start); - let arg_map = rev_map.map_ranges(&arg_token_map, arg_range, shift); - let def_map = rev_map.map_ranges(&def_token_map, def_range, 0); - - let arg_file = loc.ast_id.file_id; - let def_file = loc.def.ast_id.file_id; - - Some(ExpansionInfo { arg_file, def_file, arg_map, def_map }) + Some(ExpansionInfo { arg_start, def_start, macro_arg, macro_def, exp_map, shift }) } } } @@ -143,28 +140,30 @@ impl MacroCallId { #[derive(Debug, Clone, PartialEq, Eq)] /// ExpansionInfo mainly describes how to map text range between src and expanded macro pub struct ExpansionInfo { - pub(crate) arg_file: HirFileId, - pub(crate) def_file: HirFileId, + pub(crate) arg_start: (HirFileId, TextUnit), + pub(crate) def_start: (HirFileId, TextUnit), + pub(crate) shift: u32, - pub(crate) arg_map: Vec<(TextRange, TextRange)>, - pub(crate) def_map: Vec<(TextRange, TextRange)>, + pub(crate) macro_def: Arc<(mbe::MacroRules, mbe::TokenMap)>, + pub(crate) macro_arg: Arc<(tt::Subtree, mbe::TokenMap)>, + pub(crate) exp_map: Arc, } impl ExpansionInfo { pub fn find_range(&self, from: TextRange) -> Option<(HirFileId, TextRange)> { - for (src, dest) in &self.arg_map { - if src.is_subrange(&from) { - return Some((self.arg_file, *dest)); - } + fn look_in_rev_map(exp_map: &mbe::RevTokenMap, from: TextRange) -> Option { + exp_map.ranges.iter().find(|&it| it.0.is_subrange(&from)).map(|it| it.1) } - for (src, dest) in &self.def_map { - if src.is_subrange(&from) { - return Some((self.def_file, *dest)); - } - } + let token_id = look_in_rev_map(&self.exp_map, from)?; + let (token_map, file_offset, token_id) = if token_id.0 >= self.shift { + (&self.macro_arg.1, self.arg_start, tt::TokenId(token_id.0 - self.shift).into()) + } else { + (&self.macro_def.1, self.def_start, token_id) + }; - None + let range = token_map.relative_range_of(token_id)?; + Some((file_offset.0, TextRange::offset_len(range.start() + file_offset.1, range.len()))) } } diff --git a/crates/ra_ide_api/src/display/navigation_target.rs b/crates/ra_ide_api/src/display/navigation_target.rs index b77e192313..1bf81e7d51 100644 --- a/crates/ra_ide_api/src/display/navigation_target.rs +++ b/crates/ra_ide_api/src/display/navigation_target.rs @@ -36,7 +36,7 @@ fn find_range_from_node( ) -> (FileId, TextRange) { let text_range = node.text_range(); let (file_id, text_range) = src - .parent_expansion(db) + .expansion_info(db) .and_then(|expansion_info| expansion_info.find_range(text_range)) .unwrap_or((src, text_range)); diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index a49e63ace4..9653f7fef7 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -23,7 +23,7 @@ pub struct TokenMap { /// Maps relative range of the expanded syntax node to `tt::TokenId` #[derive(Debug, PartialEq, Eq, Default)] pub struct RevTokenMap { - ranges: Vec<(TextRange, tt::TokenId)>, + pub ranges: Vec<(TextRange, tt::TokenId)>, } /// Convert the syntax tree (what user has written) to a `TokenTree` (what macro @@ -121,36 +121,6 @@ impl RevTokenMap { fn add(&mut self, relative_range: TextRange, token_id: tt::TokenId) { self.ranges.push((relative_range, token_id.clone())) } - - /// Map a given token map to (Expanded syntax node, Input tokens) text-ranges pair - /// - /// This function do the following things: - /// - /// 1. Undo the increment of token-id `shift`: - /// When we output a token from from macro argument, we increased its id - /// by `shift` (so it's guaranteed to not to collide with anything from the definition) - /// We undo the increment here to rollback to its original token id. - /// 2. Offset the input tokens (`to`) by `parent` text-range: - /// We transforms the input tokens text-ranges from relative to original first token - /// to parent text-range - /// 3. Maps expanded tokens text-ranges to parent text-ranges - /// - pub fn map_ranges( - &self, - to: &TokenMap, - parent: TextRange, - shift: u32, - ) -> Vec<(TextRange, TextRange)> { - self.ranges - .iter() - .filter_map(|(r, tid)| { - let adjusted_id = tt::TokenId(tid.0.checked_sub(shift)?); - let to_range = to.relative_range_of(adjusted_id)?; - - Some((*r, TextRange::offset_len(to_range.start() + parent.start(), to_range.len()))) - }) - .collect() - } } /// Returns the textual content of a doc comment block as a quoted string