diff --git a/crates/ide/src/syntax_highlighting/highlight.rs b/crates/ide/src/syntax_highlighting/highlight.rs index e04fd5a7c7..f061e8637e 100644 --- a/crates/ide/src/syntax_highlighting/highlight.rs +++ b/crates/ide/src/syntax_highlighting/highlight.rs @@ -30,7 +30,7 @@ pub(super) fn token(sema: &Semantics, token: SyntaxToken) -> Optio INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => { SymbolKind::Field.into() } - INT_NUMBER | FLOAT_NUMBER => HlTag::NumericLiteral.into(), + INT_NUMBER | FLOAT_NUMBER_PART => HlTag::NumericLiteral.into(), BYTE => HlTag::ByteLiteral.into(), CHAR => HlTag::CharLiteral.into(), IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => { diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html index 60bc290121..329184730e 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html @@ -119,13 +119,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd println!("Hello {:05}!", 5); println!("Hello {:05}!", -5); println!("{:#010x}!", 27); - println!("Hello {0} is {1:.5}", "x", 0.01); - println!("Hello {1} is {2:.0$}", 5, "x", 0.01); - println!("Hello {0} is {2:.1$}", "x", 5, 0.01); - println!("Hello {} is {:.*}", "x", 5, 0.01); - println!("Hello {} is {2:.*}", "x", 5, 0.01); - println!("Hello {} is {number:.prec$}", "x", prec = 5, number = 0.01); - println!("{}, `{name:.*}` has 3 fractional digits", "Hello", 3, name=1234.56); + println!("Hello {0} is {1:.5}", "x", 0.01); + println!("Hello {1} is {2:.0$}", 5, "x", 0.01); + println!("Hello {0} is {2:.1$}", "x", 5, 0.01); + println!("Hello {} is {:.*}", "x", 5, 0.01); + println!("Hello {} is {2:.*}", "x", 5, 0.01); + println!("Hello {} is {number:.prec$}", "x", prec = 5, number = 0.01); + println!("{}, `{name:.*}` has 3 fractional digits", "Hello", 3, name=1234.56); println!("{}, `{name:.*}` has 3 characters", "Hello", 3, name="1234.56"); println!("{}, `{name:>8.*}` has 3 right-aligned characters", "Hello", 3, name="1234.56"); println!("Hello {{}}"); diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 21a0aa4284..8b8577986d 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -260,6 +260,35 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { IDENT => make_leaf!(Ident), UNDERSCORE => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident), + FLOAT_NUMBER_PART => { + // Reassemble a split-up float token. + let mut range = range; + let mut text = token.to_text(conv).to_string(); + if let Some(dot) = conv.peek() { + if dot.kind(conv) == DOT { + let (_, dot_range) = conv.bump().unwrap(); + text += &*dot.to_text(conv); + range = TextRange::new(range.start(), dot_range.end()); + + if let Some(tail) = conv.peek() { + if tail.kind(conv) == FLOAT_NUMBER_PART { + let (_, tail_range) = conv.bump().unwrap(); + text += &*tail.to_text(conv); + range = TextRange::new(range.start(), tail_range.end()); + } + } + } + } + + result.push( + tt::Leaf::from(tt::Literal { + id: conv.id_alloc().alloc(range, synth_id), + text: text.into(), + }) + .into(), + ); + continue; + } k if k.is_literal() => make_leaf!(Literal), LIFETIME_IDENT => { let char_unit = TextSize::of('\''); diff --git a/crates/mbe/src/to_parser_input.rs b/crates/mbe/src/to_parser_input.rs index 6faa147218..958f643320 100644 --- a/crates/mbe/src/to_parser_input.rs +++ b/crates/mbe/src/to_parser_input.rs @@ -35,15 +35,13 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input { let is_negated = lit.text.starts_with('-'); let inner_text = &lit.text[if is_negated { 1 } else { 0 }..]; - let kind = parser::LexedStr::single_token(inner_text) - .map(|(kind, _error)| kind) - .filter(|kind| { - kind.is_literal() - && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER)) - }) - .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit)); - - res.push(kind); + let lexed_str = parser::LexedStr::new(inner_text); + if lexed_str.is_empty() { + panic!("failed to convert literal: {:?}", lit); + } + for i in 0..lexed_str.len() { + res.push(lexed_str.kind(i)); + } } tt::Leaf::Ident(ident) => match ident.text.as_ref() { "_" => res.push(T![_]), diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs index 4efbf9a606..45d9b2e4e0 100644 --- a/crates/parser/src/grammar.rs +++ b/crates/parser/src/grammar.rs @@ -318,7 +318,7 @@ fn name_ref(p: &mut Parser) { } fn name_ref_or_index(p: &mut Parser) { - assert!(p.at(IDENT) || p.at(INT_NUMBER)); + assert!(p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART)); let m = p.start(); p.bump_any(); m.complete(p, NAME_REF); diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs index b063c73a9d..85f53dd237 100644 --- a/crates/parser/src/grammar/expressions.rs +++ b/crates/parser/src/grammar/expressions.rs @@ -475,11 +475,8 @@ fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { assert!(p.at(T![.])); let m = lhs.precede(p); p.bump(T![.]); - if p.at(IDENT) || p.at(INT_NUMBER) { + if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) { name_ref_or_index(p); - } else if p.at(FLOAT_NUMBER) { - // FIXME: How to recover and instead parse INT + T![.]? - p.bump_any(); } else { p.error("expected field name or number"); } diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index 37f8a7e3b7..07b0a2aee5 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -17,7 +17,7 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[ T![true], T![false], INT_NUMBER, - FLOAT_NUMBER, + FLOAT_NUMBER_PART, BYTE, CHAR, STRING, @@ -29,11 +29,19 @@ pub(crate) fn literal(p: &mut Parser) -> Option { return None; } let m = p.start(); - if p.at(FLOAT_NUMBER) { + if p.at(FLOAT_NUMBER_PART) { + // Floats can be up to 3 tokens: 2 `FLOAT_NUMBER_PART`s separated by 1 `DOT` let f = p.start(); - p.bump(FLOAT_NUMBER); + p.bump(FLOAT_NUMBER_PART); + if p.at(DOT) { + p.bump(DOT); + if p.at(FLOAT_NUMBER_PART) { + p.bump(FLOAT_NUMBER_PART); + } + } f.complete(p, FLOAT_LITERAL); } else { + // Everything else is just one token. p.bump_any(); } Some(m.complete(p, LITERAL)) diff --git a/crates/parser/src/grammar/patterns.rs b/crates/parser/src/grammar/patterns.rs index 1f622b32e5..c16bd8d0c7 100644 --- a/crates/parser/src/grammar/patterns.rs +++ b/crates/parser/src/grammar/patterns.rs @@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option { } fn is_literal_pat_start(p: &Parser) -> bool { - p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER) + p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART) || p.at_ts(expressions::LITERAL_FIRST) } diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs index f4b9988eac..fae5d8884b 100644 --- a/crates/parser/src/lexed_str.rs +++ b/crates/parser/src/lexed_str.rs @@ -177,7 +177,7 @@ impl<'a> Converter<'a> { rustc_lexer::TokenKind::RawIdent => IDENT, rustc_lexer::TokenKind::Literal { kind, .. } => { - self.extend_literal(token_text.len(), kind); + self.extend_literal(token_text, kind); return; } @@ -223,7 +223,7 @@ impl<'a> Converter<'a> { self.push(syntax_kind, token_text.len(), err); } - fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) { + fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) { let mut err = ""; let syntax_kind = match *kind { @@ -237,7 +237,22 @@ impl<'a> Converter<'a> { if empty_exponent { err = "Missing digits after the exponent symbol"; } - FLOAT_NUMBER + + // In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0` + // is lexed as a float, we split floats that contain a `.` into 3 tokens. + if let Some((before, after)) = token_text.split_once('.') { + let err = if err.is_empty() { None } else { Some(err) }; + if !before.is_empty() { + self.push(FLOAT_NUMBER_PART, before.len(), None); + } + self.push(DOT, 1, None); + if !after.is_empty() { + self.push(FLOAT_NUMBER_PART, after.len(), err); + } + return; + } + + FLOAT_NUMBER_PART } rustc_lexer::LiteralKind::Char { terminated } => { if !terminated { @@ -295,6 +310,6 @@ impl<'a> Converter<'a> { }; let err = if err.is_empty() { None } else { Some(err) }; - self.push(syntax_kind, len, err); + self.push(syntax_kind, token_text.len(), err); } } diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs index 47bf4ba92b..31fe1b2aba 100644 --- a/crates/parser/src/syntax_kind/generated.rs +++ b/crates/parser/src/syntax_kind/generated.rs @@ -110,7 +110,7 @@ pub enum SyntaxKind { RAW_KW, MACRO_RULES_KW, INT_NUMBER, - FLOAT_NUMBER, + FLOAT_NUMBER_PART, CHAR, BYTE, STRING, @@ -287,7 +287,7 @@ impl SyntaxKind { } pub fn is_literal(self) -> bool { match self { - INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING => true, + INT_NUMBER | FLOAT_NUMBER_PART | CHAR | BYTE | STRING | BYTE_STRING => true, _ => false, } } @@ -387,5 +387,5 @@ impl SyntaxKind { } } #[macro_export] -macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; [float_number] => { $ crate :: SyntaxKind :: FLOAT_NUMBER } ; } +macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; [float_number_part] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_PART } ; } pub use T; diff --git a/crates/parser/test_data/lexer/err/empty_exponent.rast b/crates/parser/test_data/lexer/err/empty_exponent.rast index af03d73ced..d0a268b8c1 100644 --- a/crates/parser/test_data/lexer/err/empty_exponent.rast +++ b/crates/parser/test_data/lexer/err/empty_exponent.rast @@ -1,14 +1,14 @@ -FLOAT_NUMBER "0e" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "0e" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "0E" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "0E" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER "42e+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42e-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42E+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42E-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" INT_NUMBER "42" DOT "." @@ -30,19 +30,35 @@ DOT "." IDENT "E" MINUS "-" WHITESPACE "\n\n" -FLOAT_NUMBER "42.2e+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2e-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2E+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2E-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER "42.2e+f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2e-f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2E+f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2E-f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" diff --git a/crates/parser/test_data/lexer/ok/numbers.rast b/crates/parser/test_data/lexer/ok/numbers.rast index 8d13c3f610..689422933c 100644 --- a/crates/parser/test_data/lexer/ok/numbers.rast +++ b/crates/parser/test_data/lexer/ok/numbers.rast @@ -4,7 +4,8 @@ INT_NUMBER "00" WHITESPACE " " INT_NUMBER "0_" WHITESPACE " " -FLOAT_NUMBER "0." +FLOAT_NUMBER_PART "0" +DOT "." WHITESPACE " " INT_NUMBER "0z" WHITESPACE "\n" @@ -20,11 +21,13 @@ INT_NUMBER "001279" WHITESPACE " " INT_NUMBER "0_1279" WHITESPACE " " -FLOAT_NUMBER "0.1279" +FLOAT_NUMBER_PART "0" +DOT "." +FLOAT_NUMBER_PART "1279" WHITESPACE " " -FLOAT_NUMBER "0e1279" +FLOAT_NUMBER_PART "0e1279" WHITESPACE " " -FLOAT_NUMBER "0E1279" +FLOAT_NUMBER_PART "0E1279" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -37,7 +40,7 @@ IDENT "foo" L_PAREN "(" R_PAREN ")" WHITESPACE "\n" -FLOAT_NUMBER "0e+1" +FLOAT_NUMBER_PART "0e+1" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -45,13 +48,19 @@ IDENT "e" PLUS "+" INT_NUMBER "1" WHITESPACE "\n" -FLOAT_NUMBER "0.0E-2" +FLOAT_NUMBER_PART "0" +DOT "." +FLOAT_NUMBER_PART "0E-2" WHITESPACE "\n" -FLOAT_NUMBER "0___0.10000____0000e+111__" +FLOAT_NUMBER_PART "0___0" +DOT "." +FLOAT_NUMBER_PART "10000____0000e+111__" WHITESPACE "\n" INT_NUMBER "1i64" WHITESPACE " " -FLOAT_NUMBER "92.0f32" +FLOAT_NUMBER_PART "92" +DOT "." +FLOAT_NUMBER_PART "0f32" WHITESPACE " " INT_NUMBER "11__s" WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast index 4064a7a1ff..f57e58a4b7 100644 --- a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast +++ b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast @@ -32,7 +32,9 @@ SOURCE_FILE INT_NUMBER "1" COMMA "," WHITESPACE " " - FLOAT_NUMBER "2.0" + FLOAT_NUMBER_PART "2" + DOT "." + FLOAT_NUMBER_PART "0" WHITESPACE "\n " R_CURLY "}" WHITESPACE " " diff --git a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast index 065af27f10..367aff324a 100644 --- a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast +++ b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast @@ -58,7 +58,9 @@ SOURCE_FILE WHITESPACE " " LITERAL FLOAT_LITERAL - FLOAT_NUMBER "2.0" + FLOAT_NUMBER_PART "2" + DOT "." + FLOAT_NUMBER_PART "0" SEMICOLON ";" WHITESPACE "\n " LET_STMT diff --git a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast index 903a1507d0..5fd9271c62 100644 --- a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast +++ b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast @@ -20,7 +20,9 @@ SOURCE_FILE METHOD_CALL_EXPR LITERAL FLOAT_LITERAL - FLOAT_NUMBER "1.0f32" + FLOAT_NUMBER_PART "1" + DOT "." + FLOAT_NUMBER_PART "0f32" DOT "." NAME_REF IDENT "floor" @@ -42,7 +44,9 @@ SOURCE_FILE METHOD_CALL_EXPR LITERAL FLOAT_LITERAL - FLOAT_NUMBER "1.0f32" + FLOAT_NUMBER_PART "1" + DOT "." + FLOAT_NUMBER_PART "0f32" DOT "." NAME_REF IDENT "floor" diff --git a/crates/syntax/rust.ungram b/crates/syntax/rust.ungram index 4b3e483bff..8e4f07d3b5 100644 --- a/crates/syntax/rust.ungram +++ b/crates/syntax/rust.ungram @@ -373,7 +373,9 @@ Literal = ) FloatLiteral = - 'float_number' + 'float_number_part' + '.'? + 'float_number_part'? PathExpr = Attr* Path diff --git a/crates/syntax/src/ast/expr_ext.rs b/crates/syntax/src/ast/expr_ext.rs index f4bff80c4c..11d81d4778 100644 --- a/crates/syntax/src/ast/expr_ext.rs +++ b/crates/syntax/src/ast/expr_ext.rs @@ -345,7 +345,7 @@ impl ast::Literal { impl ast::FloatLiteral { pub fn suffix(&self) -> Option { - ast::FloatNumber::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string()) + ast::FloatNumberPart::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string()) } } diff --git a/crates/syntax/src/ast/generated/nodes.rs b/crates/syntax/src/ast/generated/nodes.rs index b294849937..7c82372aa2 100644 --- a/crates/syntax/src/ast/generated/nodes.rs +++ b/crates/syntax/src/ast/generated/nodes.rs @@ -1090,9 +1090,10 @@ pub struct FloatLiteral { pub(crate) syntax: SyntaxNode, } impl FloatLiteral { - pub fn float_number_token(&self) -> Option { - support::token(&self.syntax, T![float_number]) + pub fn float_number_part_token(&self) -> Option { + support::token(&self.syntax, T![float_number_part]) } + pub fn dot_token(&self) -> Option { support::token(&self.syntax, T![.]) } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/crates/syntax/src/ast/generated/tokens.rs b/crates/syntax/src/ast/generated/tokens.rs index a3209c5abd..1e1a55e326 100644 --- a/crates/syntax/src/ast/generated/tokens.rs +++ b/crates/syntax/src/ast/generated/tokens.rs @@ -112,16 +112,16 @@ impl AstToken for IntNumber { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct FloatNumber { +pub struct FloatNumberPart { pub(crate) syntax: SyntaxToken, } -impl std::fmt::Display for FloatNumber { +impl std::fmt::Display for FloatNumberPart { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(&self.syntax, f) } } -impl AstToken for FloatNumber { - fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER } +impl AstToken for FloatNumberPart { + fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER_PART } fn cast(syntax: SyntaxToken) -> Option { if Self::can_cast(syntax.kind()) { Some(Self { syntax }) diff --git a/crates/syntax/src/ast/node_ext.rs b/crates/syntax/src/ast/node_ext.rs index f2153ca921..5b19b5ed2c 100644 --- a/crates/syntax/src/ast/node_ext.rs +++ b/crates/syntax/src/ast/node_ext.rs @@ -555,7 +555,9 @@ impl ast::FieldExpr { self.syntax .children_with_tokens() // FIXME: Accepting floats here to reject them in validation later - .find(|c| c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER) + .find(|c| { + c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER_PART + }) .as_ref() .and_then(SyntaxElement::as_token) .cloned() diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs index 3063396b44..5f2e7231d9 100644 --- a/crates/syntax/src/ast/token_ext.rs +++ b/crates/syntax/src/ast/token_ext.rs @@ -321,7 +321,7 @@ impl ast::IntNumber { } } -impl ast::FloatNumber { +impl ast::FloatNumberPart { pub fn suffix(&self) -> Option<&str> { let text = self.text(); let mut indices = text.char_indices(); diff --git a/crates/syntax/src/tests/ast_src.rs b/crates/syntax/src/tests/ast_src.rs index 964bd6c8b3..0a0632da74 100644 --- a/crates/syntax/src/tests/ast_src.rs +++ b/crates/syntax/src/tests/ast_src.rs @@ -71,7 +71,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield", ], contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"], - literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"], + literals: &["INT_NUMBER", "FLOAT_NUMBER_PART", "CHAR", "BYTE", "STRING", "BYTE_STRING"], tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"], nodes: &[ "SOURCE_FILE", diff --git a/crates/syntax/src/tests/sourcegen_ast.rs b/crates/syntax/src/tests/sourcegen_ast.rs index e18f575e37..eb6a5f63ea 100644 --- a/crates/syntax/src/tests/sourcegen_ast.rs +++ b/crates/syntax/src/tests/sourcegen_ast.rs @@ -462,7 +462,7 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String { [lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT }; [ident] => { $crate::SyntaxKind::IDENT }; [shebang] => { $crate::SyntaxKind::SHEBANG }; - [float_number] => { $crate::SyntaxKind::FLOAT_NUMBER }; + [float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART }; } pub use T; }; @@ -586,7 +586,7 @@ impl Field { fn lower(grammar: &Grammar) -> AstSrc { let mut res = AstSrc { - tokens: "Whitespace Comment String ByteString IntNumber FloatNumber Char Byte Ident" + tokens: "Whitespace Comment String ByteString IntNumber FloatNumberPart Char Byte Ident" .split_ascii_whitespace() .map(|it| it.to_string()) .collect::>(),