Address PR comments

crates/proc-macro-srv/src/server.rs

@@ -17,7 +17,7 @@ pub mod rust_analyzer_span;
 mod symbol;
 pub mod token_id;
 pub use symbol::*;
-use syntax::ast::{self, HasModuleItem, IsString};
+use syntax::ast::{self, IsString};
 use tt::Spacing;
 
 fn delim_to_internal<S>(d: proc_macro::Delimiter, span: bridge::DelimSpan<S>) -> tt::Delimiter<S> {
@@ -56,50 +56,29 @@ fn spacing_to_external(spacing: Spacing) -> proc_macro::Spacing {
 }
 
 fn literal_to_external(literal_kind: ast::LiteralKind) -> Option<proc_macro::bridge::LitKind> {
-    Some(match literal_kind {
+    match literal_kind {
-        ast::LiteralKind::String(data) => {
+        ast::LiteralKind::String(data) => Some(if data.is_raw() {
-            if data.is_raw() {
+            bridge::LitKind::StrRaw(data.raw_delimiter_count()?)
-                bridge::LitKind::StrRaw(data.raw_delimiter_count()?)
+        } else {
-            } else {
+            bridge::LitKind::Str
-                bridge::LitKind::Str
+        }),
-            }
-        }
-        ast::LiteralKind::ByteString(data) => {
-            if data.is_raw() {
-                bridge::LitKind::ByteStrRaw(data.raw_delimiter_count()?)
-            } else {
-                bridge::LitKind::ByteStr
-            }
-        }
-        ast::LiteralKind::CString(data) => {
-            if data.is_raw() {
-                bridge::LitKind::CStrRaw(data.raw_delimiter_count()?)
-            } else {
-                bridge::LitKind::CStr
-            }
-        }
-        ast::LiteralKind::IntNumber(_) => bridge::LitKind::Integer,
-        ast::LiteralKind::FloatNumber(_) => bridge::LitKind::Float,
-        ast::LiteralKind::Char(_) => bridge::LitKind::Char,
-        ast::LiteralKind::Byte(_) => bridge::LitKind::Byte,
-        ast::LiteralKind::Bool(_) => unreachable!(),
-    })
-}
 
-fn str_to_lit_node(input: &str) -> Option<ast::Literal> {
+        ast::LiteralKind::ByteString(data) => Some(if data.is_raw() {
-    let input = input.trim();
+            bridge::LitKind::ByteStrRaw(data.raw_delimiter_count()?)
-    let source_code = format!("fn f() {{ let _ = {input}; }}");
+        } else {
-
+            bridge::LitKind::ByteStr
-    let parse = ast::SourceFile::parse(&source_code);
+        }),
-    let file = parse.tree();
+        ast::LiteralKind::CString(data) => Some(if data.is_raw() {
-
+            bridge::LitKind::CStrRaw(data.raw_delimiter_count()?)
-    let ast::Item::Fn(func) = file.items().next()? else { return None };
+        } else {
-    let ast::Stmt::LetStmt(stmt) = func.body()?.stmt_list()?.statements().next()? else {
+            bridge::LitKind::CStr
-        return None;
+        }),
-    };
+        ast::LiteralKind::IntNumber(_) => Some(bridge::LitKind::Integer),
-    let ast::Expr::Literal(lit) = stmt.initializer()? else { return None };
+        ast::LiteralKind::FloatNumber(_) => Some(bridge::LitKind::Float),
-
+        ast::LiteralKind::Char(_) => Some(bridge::LitKind::Char),
-    Some(lit)
+        ast::LiteralKind::Byte(_) => Some(bridge::LitKind::Byte),
+        ast::LiteralKind::Bool(_) => None,
+    }
 }
 
 struct LiteralFormatter<S>(bridge::Literal<S, Symbol>);

crates/proc-macro-srv/src/server/rust_analyzer_span.rs

@@ -13,11 +13,11 @@ use std::{
 use ::tt::{TextRange, TextSize};
 use proc_macro::bridge::{self, server};
 use span::{Span, FIXUP_ERASED_FILE_AST_ID_MARKER};
-use syntax::ast;
+use syntax::ast::{self, IsString};
 
 use crate::server::{
-    delim_to_external, delim_to_internal, literal_to_external, str_to_lit_node,
+    delim_to_external, delim_to_internal, literal_to_external, token_stream::TokenStreamBuilder,
-    token_stream::TokenStreamBuilder, LiteralFormatter, Symbol, SymbolInternerRef, SYMBOL_INTERNER,
+    LiteralFormatter, Symbol, SymbolInternerRef, SYMBOL_INTERNER,
 };
 mod tt {
     pub use ::tt::*;
@@ -71,7 +71,8 @@ impl server::FreeFunctions for RaSpanServer {
         &mut self,
         s: &str,
     ) -> Result<bridge::Literal<Self::Span, Self::Symbol>, ()> {
-        let literal = str_to_lit_node(s).ok_or(())?;
+        let literal = ast::Literal::parse(s);
+        let literal = literal.tree();
 
         let kind = literal_to_external(literal.kind()).ok_or(())?;
 
@@ -80,12 +81,22 @@ impl server::FreeFunctions for RaSpanServer {
             ast::LiteralKind::FloatNumber(num) => num.suffix().map(ToString::to_string),
             ast::LiteralKind::IntNumber(num) => num.suffix().map(ToString::to_string),
             _ => None,
-        }
+        };
-        .map(|suffix| Symbol::intern(self.interner, &suffix));
+
+        let text = match literal.kind() {
+            ast::LiteralKind::String(data) => data.text_without_quotes().to_string(),
+            ast::LiteralKind::ByteString(data) => data.text_without_quotes().to_string(),
+            ast::LiteralKind::CString(data) => data.text_without_quotes().to_string(),
+            _ => s.to_string(),
+        };
+        let text = if let Some(ref suffix) = suffix { text.strip_suffix(suffix) } else { None }
+            .unwrap_or(&text);
+
+        let suffix = suffix.map(|suffix| Symbol::intern(self.interner, &suffix));
 
         Ok(bridge::Literal {
             kind,
-            symbol: Symbol::intern(self.interner, s),
+            symbol: Symbol::intern(self.interner, text),
             suffix,
             span: self.call_site,
         })

crates/proc-macro-srv/src/server/token_id.rs

@@ -6,11 +6,11 @@ use std::{
 };
 
 use proc_macro::bridge::{self, server};
-use syntax::ast;
+use syntax::ast::{self, IsString};
 
 use crate::server::{
-    delim_to_external, delim_to_internal, literal_to_external, str_to_lit_node,
+    delim_to_external, delim_to_internal, literal_to_external, token_stream::TokenStreamBuilder,
-    token_stream::TokenStreamBuilder, LiteralFormatter, Symbol, SymbolInternerRef, SYMBOL_INTERNER,
+    LiteralFormatter, Symbol, SymbolInternerRef, SYMBOL_INTERNER,
 };
 mod tt {
     pub use proc_macro_api::msg::TokenId;
@@ -63,7 +63,8 @@ impl server::FreeFunctions for TokenIdServer {
         &mut self,
         s: &str,
     ) -> Result<bridge::Literal<Self::Span, Self::Symbol>, ()> {
-        let literal = str_to_lit_node(s).ok_or(())?;
+        let literal = ast::Literal::parse(s);
+        let literal = literal.tree();
 
         let kind = literal_to_external(literal.kind()).ok_or(())?;
 
@@ -72,12 +73,22 @@ impl server::FreeFunctions for TokenIdServer {
             ast::LiteralKind::FloatNumber(num) => num.suffix().map(ToString::to_string),
             ast::LiteralKind::IntNumber(num) => num.suffix().map(ToString::to_string),
             _ => None,
-        }
+        };
-        .map(|suffix| Symbol::intern(self.interner, &suffix));
+
+        let text = match literal.kind() {
+            ast::LiteralKind::String(data) => data.text_without_quotes().to_string(),
+            ast::LiteralKind::ByteString(data) => data.text_without_quotes().to_string(),
+            ast::LiteralKind::CString(data) => data.text_without_quotes().to_string(),
+            _ => s.to_string(),
+        };
+        let text = if let Some(ref suffix) = suffix { text.strip_suffix(suffix) } else { None }
+            .unwrap_or(&text);
+
+        let suffix = suffix.map(|suffix| Symbol::intern(self.interner, &suffix));
 
         Ok(bridge::Literal {
             kind,
-            symbol: Symbol::intern(self.interner, s),
+            symbol: Symbol::intern(self.interner, text),
             suffix,
             span: self.call_site,
         })

crates/syntax/src/lib.rs

@@ -187,6 +187,25 @@ impl SourceFile {
     }
 }
 
+impl ast::Literal {
+    pub fn parse(text: &str) -> Parse<ast::Literal> {
+        let lexed = parser::LexedStr::new(text);
+        let parser_input = lexed.to_input();
+        let parser_output = parser::TopEntryPoint::Expr.parse(&parser_input);
+        let (green, mut errors, _) = parsing::build_tree(lexed, parser_output);
+        let root = SyntaxNode::new_root(green.clone());
+
+        errors.extend(validation::validate(&root));
+
+        assert_eq!(root.kind(), SyntaxKind::LITERAL);
+        Parse {
+            green,
+            errors: if errors.is_empty() { None } else { Some(errors.into()) },
+            _ty: PhantomData,
+        }
+    }
+}
+
 impl ast::TokenTree {
     pub fn reparse_as_comma_separated_expr(self) -> Parse<ast::MacroEagerInput> {
         let tokens = self.syntax().descendants_with_tokens().filter_map(NodeOrToken::into_token);