internal: Lay basic ground work for standalone mbe tests

Cargo.lock

@@ -1029,6 +1029,7 @@ version = "0.0.0"
 dependencies = [
  "arrayvec",
  "cov-mark",
+ "expect-test",
  "intern",
  "parser",
  "rustc-hash",

crates/hir-def/src/macro_expansion_tests/mbe.rs

@@ -389,7 +389,7 @@ m! { foo# bar }
 
 m! { Foo,# Bar }
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! m {
     ($($i:ident),*) => ($(mod $i {} )*);
     ($($i:ident)#*) => ($(fn $i() {} )*);
@@ -404,27 +404,29 @@ fn bar() {}
 
 struct Foo;
 struct Bar;
-"##]],
+"#]],
     );
 }
 
 #[test]
 fn test_match_group_pattern_with_multiple_defs() {
+    // FIXME: The pretty printer breaks by leaving whitespace here, +syntaxctxt is used to avoid that
     check(
         r#"
 macro_rules! m {
     ($($i:ident),*) => ( impl Bar { $(fn $i() {})* } );
 }
+// +syntaxctxt
 m! { foo, bar }
 "#,
         expect![[r#"
 macro_rules! m {
     ($($i:ident),*) => ( impl Bar { $(fn $i() {})* } );
 }
-impl Bar {
+impl#\1# Bar#\1# {#\1#
-    fn foo() {}
+    fn#\1# foo#\0#(#\1#)#\1# {#\1#}#\1#
-    fn bar() {}
+    fn#\1# bar#\0#(#\1#)#\1# {#\1#}#\1#
-}
+}#\1#
 "#]],
     );
 }
@@ -480,12 +482,12 @@ macro_rules! m {
 }
 m!{#abc}
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! m {
     ($($i:ident)* #abc) => ( fn baz() { $($i ();)* } );
 }
 fn baz() {}
-"##]],
+"#]],
     )
 }
 
@@ -1189,13 +1191,13 @@ macro_rules! m {
 m! { cfg(target_os = "windows") }
 m! { hello::world }
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! m {
     ($m:meta) => ( #[$m] fn bar() {} )
 }
 #[cfg(target_os = "windows")] fn bar() {}
 #[hello::world] fn bar() {}
-"##]],
+"#]],
     );
 }
 
@@ -1213,7 +1215,7 @@ m! {
     */
 }
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! m {
     ($(#[$m:meta])+) => ( $(#[$m])+ fn bar() {} )
 }
@@ -1221,7 +1223,7 @@ macro_rules! m {
 #[doc = r"
         MultiLines Doc
     "] fn bar() {}
-"##]],
+"#]],
     );
 }
 
@@ -1234,12 +1236,12 @@ macro_rules! m {
 }
 m! { #[doc = concat!("The `", "bla", "` lang item.")] }
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! m {
     (#[$m:meta]) => ( #[$m] fn bar() {} )
 }
 #[doc = concat!("The `", "bla", "` lang item.")] fn bar() {}
-"##]],
+"#]],
     );
 }
 
@@ -1257,7 +1259,7 @@ m! {
     */
 }
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! m {
     ($(#[$ m:meta])+) => ( $(#[$m])+ fn bar() {} )
 }
@@ -1265,7 +1267,7 @@ macro_rules! m {
 #[doc = r"
         莊生曉夢迷蝴蝶，望帝春心託杜鵑。
     "] fn bar() {}
-"##]],
+"#]],
     );
 }
 
@@ -1342,10 +1344,10 @@ fn test_tt_composite2() {
 macro_rules! m { ($($tt:tt)*) => { abs!(=> $($tt)*); } }
 m! {#}
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! m { ($($tt:tt)*) => { abs!(=> $($tt)*); } }
 abs!( = > #);
-"##]],
+"#]],
     );
 }
 

crates/hir-def/src/macro_expansion_tests/mbe/regression.rs

@@ -139,7 +139,7 @@ STRUCT!{struct D3DVSHADERCAPS2_0 {Caps: u8,}}
 
 STRUCT!{#[cfg_attr(target_arch = "x86", repr(packed))] struct D3DCONTENTPROTECTIONCAPS {Caps : u8 ,}}
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! STRUCT {
     ($(#[$attrs:meta])* struct $name:ident {
         $($field:ident: $ftype:ty,)+
@@ -194,7 +194,7 @@ impl Clone for D3DCONTENTPROTECTIONCAPS {
         }
     }
 }
-"##]],
+"#]],
     );
 }
 
@@ -214,7 +214,7 @@ macro_rules! int_base {
 }
 int_base!{Binary for isize as usize -> Binary}
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! int_base {
     ($Trait:ident for $T:ident as $U:ident -> $Radix:ident) => {
         #[stable(feature = "rust1", since = "1.0.0")]
@@ -230,7 +230,7 @@ macro_rules! int_base {
         Binary.fmt_int(*self as usize, f)
     }
 }
-"##]],
+"#]],
     );
 }
 
@@ -318,7 +318,7 @@ impl_fn_for_zst !   {
 }
 
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! impl_fn_for_zst  {
     {$( $( #[$attr: meta] )*
     struct $Name: ident impl$( <$( $lifetime : lifetime ),+> )? Fn =
@@ -410,7 +410,7 @@ impl FnOnce<(char, )> for CharEscapeDefault {
     }
 }
 
-"##]],
+"#]],
     );
 }
 
@@ -511,7 +511,7 @@ cfg_if! {
     @__apply cfg(all(not(any(not(any(target_os = "solaris", target_os = "illumos")))))),
 }
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! cfg_if {
     ($(if #[cfg($($meta:meta),*)] { $($it:item)* } )else* else { $($it2:item)* })
     => {
@@ -534,7 +534,7 @@ __cfg_if_items! {
 }
 
 
-"##]],
+"#]],
     );
 }
 
@@ -618,7 +618,7 @@ RIDL!{interface ID3D11Asynchronous(ID3D11AsynchronousVtbl): ID3D11DeviceChild(ID
     fn GetDataSize(&mut self) -> UINT
 }}
 "#,
-        expect![[r##"
+        expect![[r#"
 #[macro_export]
 macro_rules! RIDL {
     (interface $interface:ident ($vtbl:ident) : $pinterface:ident ($pvtbl:ident)
@@ -639,7 +639,7 @@ impl ID3D11Asynchronous {
         ((*self .lpVtbl).GetDataSize)(self )
     }
 }
-"##]],
+"#]],
     );
 }
 
@@ -676,7 +676,7 @@ quick_error ! (
 );
 
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! quick_error {
     (SORT [enum $name:ident $( #[$meta:meta] )*]
         items [$($( #[$imeta:meta] )*
@@ -697,7 +697,7 @@ macro_rules! quick_error {
 }
 quick_error!(ENUMINITION[enum Wrapped#[derive(Debug)]]body[]queue[ = > One: UNIT[] = > Two: TUPLE[s: String]]);
 
-"##]],
+"#]],
     )
 }
 
@@ -746,7 +746,7 @@ delegate_impl ! {
     [G, &'a mut G, deref] pub trait Data: GraphBase {@section type type NodeWeight;}
 }
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! delegate_impl {
     ([$self_type:ident, $self_wrap:ty, $self_map:ident]
      pub trait $name:ident $(: $sup:ident)* $(+ $more_sup:ident)* {
@@ -785,7 +785,7 @@ macro_rules! delegate_impl {
     }
 }
 impl <> Data for &'amut G where G: Data {}
-"##]],
+"#]],
     );
 }
 
@@ -959,14 +959,14 @@ macro_rules! with_std {
 
 with_std! {mod m;mod f;}
 "#,
-        expect![[r##"
+        expect![[r#"
 macro_rules! with_std {
     ($($i:item)*) => ($(#[cfg(feature = "std")]$i)*)
 }
 
 #[cfg(feature = "std")] mod m;
 #[cfg(feature = "std")] mod f;
-"##]],
+"#]],
     )
 }
 

crates/hir-def/src/macro_expansion_tests/mod.rs

@@ -1,6 +1,6 @@
-//! This module contains tests for macro expansion. Effectively, it covers `tt`,
+//! This module contains integration tests for macro expansion with name resolution. Effectively, it
-//! `mbe`, `proc_macro_api` and `hir_expand` crates. This might seem like a
+//! covers `tt`, `mbe`, `proc_macro_api` and `hir_expand` crates. This might seem like a  wrong
-//! wrong architecture at the first glance, but is intentional.
+//! architecture at the first glance, but is intentional.
 //!
 //! Physically, macro expansion process is intertwined with name resolution. You
 //! can not expand *just* the syntax. So, to be able to write integration tests

crates/hir-def/src/macro_expansion_tests/proc_macros.rs

@@ -16,12 +16,12 @@ fn attribute_macro_attr_censoring() {
 #[attr1] #[proc_macros::identity] #[attr2]
 struct S;
 "#,
-        expect![[r##"
+        expect![[r#"
 #[attr1] #[proc_macros::identity] #[attr2]
 struct S;
 
 #[attr1]
-#[attr2] struct S;"##]],
+#[attr2] struct S;"#]],
     );
 }
 
@@ -39,7 +39,7 @@ fn derive_censoring() {
 #[attr2]
 struct S;
 "#,
-        expect![[r##"
+        expect![[r#"
 #[attr1]
 #[derive(Foo)]
 #[derive(proc_macros::DeriveIdentity)]
@@ -49,7 +49,7 @@ struct S;
 
 #[attr1]
 #[derive(Bar)]
-#[attr2] struct S;"##]],
+#[attr2] struct S;"#]],
     );
 }
 
@@ -62,14 +62,14 @@ fn attribute_macro_syntax_completion_1() {
 #[proc_macros::identity_when_valid]
 fn foo() { bar.baz(); blub }
 "#,
-        expect![[r##"
+        expect![[r#"
 #[proc_macros::identity_when_valid]
 fn foo() { bar.baz(); blub }
 
 fn foo() {
     bar.baz();
     blub
-}"##]],
+}"#]],
     );
 }
 

crates/hir-expand/src/lib.rs

@@ -55,6 +55,7 @@ pub use crate::files::{AstId, ErasedAstId, FileRange, InFile, InMacroFile, InRea
 
 pub use mbe::{DeclarativeMacro, ValueResult};
 pub use span::{HirFileId, MacroCallId, MacroFileId};
+pub use syntax_bridge::insert_whitespace_into_node;
 
 pub mod tt {
     pub use span::Span;

crates/hir/src/lib.rs

@@ -136,6 +136,7 @@ pub use {
         },
         hygiene::{marks_rev, SyntaxContextExt},
         inert_attr_macro::AttributeTemplate,
+        insert_whitespace_into_node,
         name::Name,
         proc_macro::{ProcMacros, ProcMacrosBuilder},
         tt, ExpandResult, HirFileId, HirFileIdExt, MacroFileId, MacroFileIdExt,

crates/ide-db/src/lib.rs

@@ -36,7 +36,7 @@ pub mod generated {
 pub mod syntax_helpers {
     pub mod format_string;
     pub mod format_string_exprs;
-    pub mod insert_whitespace_into_node;
+    pub use hir::insert_whitespace_into_node;
     pub mod node_ext;
 
     pub use parser::LexedStr;

crates/ide/src/hover/tests.rs

@@ -509,13 +509,8 @@ fn main() {
                                 file_id: FileId(
                                     1,
                                 ),
-<<<<<<< HEAD
-                                full_range: 633..868,
-                                focus_range: 694..700,
-=======
                                 full_range: 4294967295..4294967295,
                                 focus_range: 4294967295..4294967295,
->>>>>>> 33e4a08d5b (minor: Reduce friction for updating minicore)
                                 name: "FnOnce",
                                 kind: Trait,
                                 container_name: "function",

crates/mbe/Cargo.toml

@@ -30,6 +30,7 @@ syntax-bridge.workspace = true
 
 [dev-dependencies]
 test-utils.workspace = true
+expect-test.workspace = true
 
 [features]
 in-rust-tree = ["parser/in-rust-tree", "tt/in-rust-tree", "syntax/in-rust-tree"]

crates/mbe/src/lib.rs

@@ -11,6 +11,8 @@ mod parser;
 
 #[cfg(test)]
 mod benchmark;
+#[cfg(test)]
+mod tests;
 
 use span::{Edition, Span, SyntaxContextId};
 use syntax_bridge::to_parser_input;

crates/mbe/src/tests.rs

@@ -0,0 +1,179 @@
+//! Tests specific to declarative macros, aka macros by example. This covers
+//! both stable `macro_rules!` macros as well as unstable `macro` macros.
+// FIXME: Move more of the nameres independent tests from
+// crates\hir-def\src\macro_expansion_tests\mod.rs to this
+use expect_test::expect;
+use span::{Edition, EditionedFileId, ErasedFileAstId, FileId, Span, SpanAnchor, SyntaxContextId};
+use stdx::format_to;
+use syntax_bridge::insert_whitespace_into_node::insert_ws_into;
+use tt::{TextRange, TextSize};
+
+use crate::DeclarativeMacro;
+
+fn check_(
+    def_edition: Edition,
+    call_edition: Edition,
+    macro2: bool,
+    decl: &str,
+    arg: &str,
+    render_debug: bool,
+    expect: expect_test::Expect,
+    parse: parser::TopEntryPoint,
+) {
+    let decl_tt = &syntax_bridge::parse_to_token_tree(
+        def_edition,
+        SpanAnchor {
+            file_id: EditionedFileId::new(FileId::from_raw(0), def_edition),
+            ast_id: ErasedFileAstId::from_raw(0),
+        },
+        SyntaxContextId::ROOT,
+        decl,
+    )
+    .unwrap();
+    let mac = if macro2 {
+        DeclarativeMacro::parse_macro2(None, decl_tt, |_| def_edition)
+    } else {
+        DeclarativeMacro::parse_macro_rules(decl_tt, |_| def_edition)
+    };
+    let call_anchor = SpanAnchor {
+        file_id: EditionedFileId::new(FileId::from_raw(1), call_edition),
+        ast_id: ErasedFileAstId::from_raw(0),
+    };
+    let arg_tt =
+        syntax_bridge::parse_to_token_tree(call_edition, call_anchor, SyntaxContextId::ROOT, arg)
+            .unwrap();
+    let res = mac.expand(
+        &arg_tt,
+        |_| (),
+        Span {
+            range: TextRange::up_to(TextSize::of(arg)),
+            anchor: call_anchor,
+            ctx: SyntaxContextId::ROOT,
+        },
+        def_edition,
+    );
+    let mut expect_res = String::new();
+    if let Some(err) = res.err {
+        format_to!(expect_res, "{err:#?}\n\n",);
+    }
+    if render_debug {
+        format_to!(expect_res, "{:#?}\n\n", res.value.0);
+    }
+    let (node, _) = syntax_bridge::token_tree_to_syntax_node(&res.value.0, parse, def_edition);
+    format_to!(expect_res, "{}", insert_ws_into(node.syntax_node()));
+    expect.assert_eq(&expect_res);
+}
+
+fn check(
+    def_edition: Edition,
+    call_edition: Edition,
+    decl: &str,
+    arg: &str,
+    expect: expect_test::Expect,
+) {
+    check_(
+        def_edition,
+        call_edition,
+        false,
+        decl,
+        arg,
+        true,
+        expect,
+        parser::TopEntryPoint::SourceFile,
+    );
+}
+
+#[test]
+fn token_mapping_smoke_test() {
+    check(
+        Edition::CURRENT,
+        Edition::CURRENT,
+        r#"
+( struct $ident:ident ) => {
+    struct $ident {
+        map: ::std::collections::HashSet<()>,
+    }
+};
+"#,
+        r#"
+struct MyTraitMap2
+"#,
+        expect![[r#"
+            SUBTREE $$ 1:0@0..20#0 1:0@0..20#0
+              IDENT   struct 0:0@34..40#0
+              IDENT   MyTraitMap2 1:0@8..19#0
+              SUBTREE {} 0:0@48..49#0 0:0@100..101#0
+                IDENT   map 0:0@58..61#0
+                PUNCH   : [alone] 0:0@61..62#0
+                PUNCH   : [joint] 0:0@63..64#0
+                PUNCH   : [alone] 0:0@64..65#0
+                IDENT   std 0:0@65..68#0
+                PUNCH   : [joint] 0:0@68..69#0
+                PUNCH   : [alone] 0:0@69..70#0
+                IDENT   collections 0:0@70..81#0
+                PUNCH   : [joint] 0:0@81..82#0
+                PUNCH   : [alone] 0:0@82..83#0
+                IDENT   HashSet 0:0@83..90#0
+                PUNCH   < [alone] 0:0@90..91#0
+                SUBTREE () 0:0@91..92#0 0:0@92..93#0
+                PUNCH   > [joint] 0:0@93..94#0
+                PUNCH   , [alone] 0:0@94..95#0
+
+            struct MyTraitMap2 {
+                map: ::std::collections::HashSet<()>,
+            }"#]],
+    );
+}
+
+#[test]
+fn token_mapping_floats() {
+    // Regression test for https://github.com/rust-lang/rust-analyzer/issues/12216
+    // (and related issues)
+    check(
+        Edition::CURRENT,
+        Edition::CURRENT,
+        r#"
+($($tt:tt)*) => {
+    $($tt)*
+};
+"#,
+        r#"
+fn main() {
+    1;
+    1.0;
+    ((1,),).0.0;
+    let x = 1;
+}
+"#,
+        expect![[r#"
+            SUBTREE $$ 1:0@0..63#0 1:0@0..63#0
+              IDENT   fn 1:0@1..3#0
+              IDENT   main 1:0@4..8#0
+              SUBTREE () 1:0@8..9#0 1:0@9..10#0
+              SUBTREE {} 1:0@11..12#0 1:0@61..62#0
+                LITERAL Integer 1 1:0@17..18#0
+                PUNCH   ; [alone] 1:0@18..19#0
+                LITERAL Float 1.0 1:0@24..27#0
+                PUNCH   ; [alone] 1:0@27..28#0
+                SUBTREE () 1:0@33..34#0 1:0@39..40#0
+                  SUBTREE () 1:0@34..35#0 1:0@37..38#0
+                    LITERAL Integer 1 1:0@35..36#0
+                    PUNCH   , [alone] 1:0@36..37#0
+                  PUNCH   , [alone] 1:0@38..39#0
+                PUNCH   . [alone] 1:0@40..41#0
+                LITERAL Float 0.0 1:0@41..44#0
+                PUNCH   ; [alone] 1:0@44..45#0
+                IDENT   let 1:0@50..53#0
+                IDENT   x 1:0@54..55#0
+                PUNCH   = [alone] 1:0@56..57#0
+                LITERAL Integer 1 1:0@58..59#0
+                PUNCH   ; [alone] 1:0@59..60#0
+
+            fn main(){
+                1;
+                1.0;
+                ((1,),).0.0;
+                let x = 1;
+            }"#]],
+    );
+}

crates/syntax-bridge/src/insert_whitespace_into_node.rs

@@ -7,8 +7,6 @@ use syntax::{
     SyntaxNode, SyntaxToken, WalkEvent, T,
 };
 
-// FIXME: It would also be cool to share logic here and in the mbe tests,
-// which are pretty unreadable at the moment.
 /// Renders a [`SyntaxNode`] with whitespace inserted between tokens that require them.
 pub fn insert_ws_into(syn: SyntaxNode) -> SyntaxNode {
     let mut indent = 0;

crates/syntax-bridge/src/lib.rs

@@ -17,6 +17,7 @@ use tt::{
     token_to_literal,
 };
 
+pub mod insert_whitespace_into_node;
 mod to_parser_input;
 pub use to_parser_input::to_parser_input;
 // FIXME: we probably should re-think  `token_tree_to_syntax_node` interfaces

crates/vfs/src/lib.rs

@@ -67,7 +67,7 @@ pub struct FileId(u32);
 // pub struct FileId(NonMaxU32);
 
 impl FileId {
-    pub const MAX: u32 = 0x7fff_ffff;
+    const MAX: u32 = 0x7fff_ffff;
 
     #[inline]
     pub const fn from_raw(raw: u32) -> FileId {