Indicate the number of float tokens in the first token

crates/parser/src/grammar.rs

@@ -39,6 +39,7 @@ mod generic_params;
 mod types;
 
 use crate::{
+    grammar::expressions::FLOAT_LITERAL_FIRST,
     parser::{CompletedMarker, Marker, Parser},
     SyntaxKind::{self, *},
     TokenSet, T,
@@ -318,9 +319,15 @@ fn name_ref(p: &mut Parser) {
 }
 
 fn name_ref_or_index(p: &mut Parser) {
-    assert!(p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART));
+    assert!(
+        p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST)
+    );
     let m = p.start();
+    if p.at_ts(FLOAT_LITERAL_FIRST) {
+        p.bump_remap(FLOAT_NUMBER_PART);
+    } else {
         p.bump_any();
+    }
     m.complete(p, NAME_REF);
 }
 

crates/parser/src/grammar/expressions.rs

@@ -3,7 +3,7 @@ mod atom;
 use super::*;
 
 pub(crate) use self::atom::{block_expr, match_arm_list};
-pub(super) use self::atom::{float_literal, literal, LITERAL_FIRST};
+pub(super) use self::atom::{float_literal, literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST};
 
 #[derive(PartialEq, Eq)]
 pub(super) enum Semicolon {
@@ -452,6 +452,9 @@ fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
 // fn foo() {
 //     x.foo();
 //     y.bar::<T>(1, 2,);
+//
+//     0e0.sin();
+//     0e0f32.sin();
 // }
 fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
     assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
@@ -477,7 +480,7 @@ fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
     assert!(p.at(T![.]));
     let m = lhs.precede(p);
     p.bump(T![.]);
-    if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) {
+    if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) {
         name_ref_or_index(p);
     } else {
         p.error("expected field name or number");

crates/parser/src/grammar/expressions/atom.rs

@@ -17,19 +17,24 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[
     T![true],
     T![false],
     INT_NUMBER,
-    FLOAT_NUMBER_PART,
+    FLOAT_NUMBER_START_0,
+    FLOAT_NUMBER_START_1,
+    FLOAT_NUMBER_START_2,
     BYTE,
     CHAR,
     STRING,
     BYTE_STRING,
 ]);
 
+pub(crate) const FLOAT_LITERAL_FIRST: TokenSet =
+    TokenSet::new(&[FLOAT_NUMBER_START_0, FLOAT_NUMBER_START_1, FLOAT_NUMBER_START_2]);
+
 pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
     if !p.at_ts(LITERAL_FIRST) {
         return None;
     }
     let m = p.start();
-    if p.at(FLOAT_NUMBER_PART) {
+    if p.at_ts(FLOAT_LITERAL_FIRST) {
         float_literal(p);
     } else {
         // Everything else is just one token.
@@ -38,15 +43,30 @@ pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
     Some(m.complete(p, LITERAL))
 }
 
+// test float_literal
+// fn f() {
+//     0.0;
+//     1.;
+//     0e0;
+//     0e0f32;
+//     1.23f64;
+// }
 pub(crate) fn float_literal(p: &mut Parser) {
-    // Floats can be up to 3 tokens: 2 `FLOAT_NUMBER_PART`s separated by 1 `DOT`
+    // Floats can be up to 3 tokens. The first token indicates how many there are.
+    // We remap the first token to `FLOAT_NUMBER_PART` so that no subsequent code has to deal with
+    // this awful, awful hack.
     let f = p.start();
-    p.bump(FLOAT_NUMBER_PART);
+    if p.at(FLOAT_NUMBER_START_0) {
-    if p.at(DOT) {
+        p.bump_remap(FLOAT_NUMBER_PART);
+    } else if p.at(FLOAT_NUMBER_START_1) {
+        p.bump_remap(FLOAT_NUMBER_PART);
+        p.bump(DOT);
+    } else if p.at(FLOAT_NUMBER_START_2) {
+        p.bump_remap(FLOAT_NUMBER_PART);
         p.bump(DOT);
-        if p.at(FLOAT_NUMBER_PART) {
         p.bump(FLOAT_NUMBER_PART);
-        }
+    } else {
+        unreachable!();
     }
     f.complete(p, FLOAT_LITERAL);
 }

crates/parser/src/grammar/items.rs

@@ -457,7 +457,7 @@ pub(crate) fn token_tree(p: &mut Parser) {
                 return;
             }
             T![')'] | T![']'] => p.err_and_bump("unmatched brace"),
-            FLOAT_NUMBER_PART => {
+            FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => {
                 float_literal(p);
             }
             _ => p.bump_any(),

crates/parser/src/lexed_str.rs

@@ -240,11 +240,16 @@ impl<'a> Converter<'a> {
 
                 // In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0`
                 // is lexed as a float, we split floats that contain a `.` into 3 tokens.
+                // To ensure that later stages can always reconstruct the token correctly, the first
+                // token in the sequence indicates the number of following tokens that are part of
+                // the float literal.
                 if let Some((before, after)) = token_text.split_once('.') {
                     let err = if err.is_empty() { None } else { Some(err) };
-                    if !before.is_empty() {
+
-                        self.push(FLOAT_NUMBER_PART, before.len(), None);
+                    assert!(!before.is_empty());
-                    }
+                    let tok =
+                        if after.is_empty() { FLOAT_NUMBER_START_1 } else { FLOAT_NUMBER_START_2 };
+                    self.push(tok, before.len(), None);
                     self.push(DOT, 1, None);
                     if !after.is_empty() {
                         self.push(FLOAT_NUMBER_PART, after.len(), err);
@@ -252,7 +257,7 @@ impl<'a> Converter<'a> {
                     return;
                 }
 
-                FLOAT_NUMBER_PART
+                FLOAT_NUMBER_START_0
             }
             rustc_lexer::LiteralKind::Char { terminated } => {
                 if !terminated {

crates/parser/test_data/lexer/err/empty_exponent.rast

@@ -1,14 +1,14 @@
-FLOAT_NUMBER_PART "0e" error: Missing digits after the exponent symbol
+FLOAT_NUMBER_START_0 "0e" error: Missing digits after the exponent symbol
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "0E" error: Missing digits after the exponent symbol
+FLOAT_NUMBER_START_0 "0E" error: Missing digits after the exponent symbol
 WHITESPACE "\n\n"
-FLOAT_NUMBER_PART "42e+" error: Missing digits after the exponent symbol
+FLOAT_NUMBER_START_0 "42e+" error: Missing digits after the exponent symbol
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "42e-" error: Missing digits after the exponent symbol
+FLOAT_NUMBER_START_0 "42e-" error: Missing digits after the exponent symbol
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "42E+" error: Missing digits after the exponent symbol
+FLOAT_NUMBER_START_0 "42E+" error: Missing digits after the exponent symbol
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "42E-" error: Missing digits after the exponent symbol
+FLOAT_NUMBER_START_0 "42E-" error: Missing digits after the exponent symbol
 WHITESPACE "\n\n"
 INT_NUMBER "42"
 DOT "."
@@ -30,35 +30,35 @@ DOT "."
 IDENT "E"
 MINUS "-"
 WHITESPACE "\n\n"
-FLOAT_NUMBER_PART "42"
+FLOAT_NUMBER_START_2 "42"
 DOT "."
 FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "42"
+FLOAT_NUMBER_START_2 "42"
 DOT "."
 FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "42"
+FLOAT_NUMBER_START_2 "42"
 DOT "."
 FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "42"
+FLOAT_NUMBER_START_2 "42"
 DOT "."
 FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol
 WHITESPACE "\n\n"
-FLOAT_NUMBER_PART "42"
+FLOAT_NUMBER_START_2 "42"
 DOT "."
 FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "42"
+FLOAT_NUMBER_START_2 "42"
 DOT "."
 FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "42"
+FLOAT_NUMBER_START_2 "42"
 DOT "."
 FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "42"
+FLOAT_NUMBER_START_2 "42"
 DOT "."
 FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol
 WHITESPACE "\n"

crates/parser/test_data/lexer/ok/numbers.rast

@@ -4,7 +4,7 @@ INT_NUMBER "00"
 WHITESPACE " "
 INT_NUMBER "0_"
 WHITESPACE " "
-FLOAT_NUMBER_PART "0"
+FLOAT_NUMBER_START_1 "0"
 DOT "."
 WHITESPACE " "
 INT_NUMBER "0z"
@@ -21,13 +21,13 @@ INT_NUMBER "001279"
 WHITESPACE " "
 INT_NUMBER "0_1279"
 WHITESPACE " "
-FLOAT_NUMBER_PART "0"
+FLOAT_NUMBER_START_2 "0"
 DOT "."
 FLOAT_NUMBER_PART "1279"
 WHITESPACE " "
-FLOAT_NUMBER_PART "0e1279"
+FLOAT_NUMBER_START_0 "0e1279"
 WHITESPACE " "
-FLOAT_NUMBER_PART "0E1279"
+FLOAT_NUMBER_START_0 "0E1279"
 WHITESPACE "\n"
 INT_NUMBER "0"
 DOT "."
@@ -40,7 +40,7 @@ IDENT "foo"
 L_PAREN "("
 R_PAREN ")"
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "0e+1"
+FLOAT_NUMBER_START_0 "0e+1"
 WHITESPACE "\n"
 INT_NUMBER "0"
 DOT "."
@@ -48,17 +48,17 @@ IDENT "e"
 PLUS "+"
 INT_NUMBER "1"
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "0"
+FLOAT_NUMBER_START_2 "0"
 DOT "."
 FLOAT_NUMBER_PART "0E-2"
 WHITESPACE "\n"
-FLOAT_NUMBER_PART "0___0"
+FLOAT_NUMBER_START_2 "0___0"
 DOT "."
 FLOAT_NUMBER_PART "10000____0000e+111__"
 WHITESPACE "\n"
 INT_NUMBER "1i64"
 WHITESPACE " "
-FLOAT_NUMBER_PART "92"
+FLOAT_NUMBER_START_2 "92"
 DOT "."
 FLOAT_NUMBER_PART "0f32"
 WHITESPACE " "

crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast

@@ -58,6 +58,32 @@ SOURCE_FILE
               COMMA ","
               R_PAREN ")"
           SEMICOLON ";"
+        WHITESPACE "\n\n    "
+        EXPR_STMT
+          METHOD_CALL_EXPR
+            LITERAL
+              FLOAT_LITERAL
+                FLOAT_NUMBER_PART "0e0"
+            DOT "."
+            NAME_REF
+              IDENT "sin"
+            ARG_LIST
+              L_PAREN "("
+              R_PAREN ")"
+          SEMICOLON ";"
+        WHITESPACE "\n    "
+        EXPR_STMT
+          METHOD_CALL_EXPR
+            LITERAL
+              FLOAT_LITERAL
+                FLOAT_NUMBER_PART "0e0f32"
+            DOT "."
+            NAME_REF
+              IDENT "sin"
+            ARG_LIST
+              L_PAREN "("
+              R_PAREN ")"
+          SEMICOLON ";"
         WHITESPACE "\n"
         R_CURLY "}"
   WHITESPACE "\n"

crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs

@@ -1,4 +1,7 @@
 fn foo() {
     x.foo();
     y.bar::<T>(1, 2,);
+
+    0e0.sin();
+    0e0f32.sin();
 }

crates/parser/test_data/parser/inline/ok/0201_float_literal.rast

@@ -0,0 +1,51 @@
+SOURCE_FILE
+  FN
+    FN_KW "fn"
+    WHITESPACE " "
+    NAME
+      IDENT "f"
+    PARAM_LIST
+      L_PAREN "("
+      R_PAREN ")"
+    WHITESPACE " "
+    BLOCK_EXPR
+      STMT_LIST
+        L_CURLY "{"
+        WHITESPACE "\n    "
+        EXPR_STMT
+          LITERAL
+            FLOAT_LITERAL
+              FLOAT_NUMBER_PART "0"
+              DOT "."
+              FLOAT_NUMBER_PART "0"
+          SEMICOLON ";"
+        WHITESPACE "\n    "
+        EXPR_STMT
+          LITERAL
+            FLOAT_LITERAL
+              FLOAT_NUMBER_PART "1"
+              DOT "."
+          SEMICOLON ";"
+        WHITESPACE "\n    "
+        EXPR_STMT
+          LITERAL
+            FLOAT_LITERAL
+              FLOAT_NUMBER_PART "0e0"
+          SEMICOLON ";"
+        WHITESPACE "\n    "
+        EXPR_STMT
+          LITERAL
+            FLOAT_LITERAL
+              FLOAT_NUMBER_PART "0e0f32"
+          SEMICOLON ";"
+        WHITESPACE "\n    "
+        EXPR_STMT
+          LITERAL
+            FLOAT_LITERAL
+              FLOAT_NUMBER_PART "1"
+              DOT "."
+              FLOAT_NUMBER_PART "23f64"
+          SEMICOLON ";"
+        WHITESPACE "\n"
+        R_CURLY "}"
+  WHITESPACE "\n"

crates/parser/test_data/parser/inline/ok/0201_float_literal.rs

@@ -0,0 +1,7 @@
+fn f() {
+    0.0;
+    1.;
+    0e0;
+    0e0f32;
+    1.23f64;
+}

crates/syntax/rust.ungram

@@ -373,7 +373,9 @@ Literal =
   )
 
 FloatLiteral =
-  'float_number_part'
+  'float_number_start_0'?
+  'float_number_start_1'?
+  'float_number_start_2'?
   '.'?
   'float_number_part'?
 

crates/syntax/src/ast/generated/nodes.rs

@@ -1090,10 +1090,19 @@ pub struct FloatLiteral {
     pub(crate) syntax: SyntaxNode,
 }
 impl FloatLiteral {
+    pub fn float_number_start_0_token(&self) -> Option<SyntaxToken> {
+        support::token(&self.syntax, T![float_number_start_0])
+    }
+    pub fn float_number_start_1_token(&self) -> Option<SyntaxToken> {
+        support::token(&self.syntax, T![float_number_start_1])
+    }
+    pub fn float_number_start_2_token(&self) -> Option<SyntaxToken> {
+        support::token(&self.syntax, T![float_number_start_2])
+    }
+    pub fn dot_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![.]) }
     pub fn float_number_part_token(&self) -> Option<SyntaxToken> {
         support::token(&self.syntax, T![float_number_part])
     }
-    pub fn dot_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![.]) }
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]

crates/syntax/src/tests/ast_src.rs

@@ -71,7 +71,17 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
         "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield",
     ],
     contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"],
-    literals: &["INT_NUMBER", "FLOAT_NUMBER_PART", "CHAR", "BYTE", "STRING", "BYTE_STRING"],
+    literals: &[
+        "INT_NUMBER",
+        "FLOAT_NUMBER_START_0",
+        "FLOAT_NUMBER_START_1",
+        "FLOAT_NUMBER_START_2",
+        "FLOAT_NUMBER_PART",
+        "CHAR",
+        "BYTE",
+        "STRING",
+        "BYTE_STRING",
+    ],
     tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"],
     nodes: &[
         "SOURCE_FILE",

crates/syntax/src/tests/sourcegen_ast.rs

@@ -463,6 +463,9 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String {
             [ident] => { $crate::SyntaxKind::IDENT };
             [shebang] => { $crate::SyntaxKind::SHEBANG };
             [float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART };
+            [float_number_start_0] => { $crate::SyntaxKind::FLOAT_NUMBER_START_0 };
+            [float_number_start_1] => { $crate::SyntaxKind::FLOAT_NUMBER_START_1 };
+            [float_number_start_2] => { $crate::SyntaxKind::FLOAT_NUMBER_START_2 };
         }
         pub use T;
     };