Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2025-01-28 04:45:05 +00:00
Code Issues Projects Releases Packages Wiki Activity

Auto merge of #12241 - jonas-schievink:does-not-float-my-boat, r=jonas-schievink

Browse source 
fix: revert float parsing "fix" to avoid macro-related panics

Reverts https://github.com/rust-lang/rust-analyzer/pull/12149 and the follow-up fixes, while keeping their tests.

https://github.com/rust-lang/rust-analyzer/pull/12149 has caused many unexpected panics related to macros, and the fixes for those are not straightforward and further complicate the MBE token conversion logic, which was already fairly hard to follow before these fixes.
This commit is contained in:
bors 2022-05-13 15:02:00 +00:00
commit 4f6b2a20fd
42 changed files with 156 additions and 610 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
crates/hir-def/src/body/lower.rs
View file

@ -972,7 +972,7 @@ impl From<ast::LiteralKind> for Literal {
} }
} }
LiteralKind::FloatNumber(lit) => { LiteralKind::FloatNumber(lit) => {
let ty = lit.suffix().and_then(|s| BuiltinFloat::from_suffix(&s)); let ty = lit.suffix().and_then(BuiltinFloat::from_suffix);
Literal::Float(Default::default(), ty) Literal::Float(Default::default(), ty)
} }
LiteralKind::ByteString(bs) => { LiteralKind::ByteString(bs) => {

8
crates/hir-def/src/macro_expansion_tests/mbe.rs
View file

@ -48,6 +48,8 @@ struct#10 MyTraitMap2#32 {#13
#[test] #[test]
fn token_mapping_floats() { fn token_mapping_floats() {
// Regression test for https://github.com/rust-lang/rust-analyzer/issues/12216
// (and related issues)
check( check(
r#" r#"
// +tokenids // +tokenids
@ -87,9 +89,9 @@ macro_rules! f {#0
// } // }
fn#19 main#20(#21)#21 {#22 fn#19 main#20(#21)#21 {#22
1#23;#24 1#23;#24
1#26.0; 1.0#25;#26
let x#31 =#22 1; let#27 x#28 =#29 1#30;#31
} }#22
"##]], "##]],

2
crates/hir-def/src/macro_expansion_tests/mbe/meta_syntax.rs
View file

@ -80,7 +80,7 @@ macro_rules! f3 { ($i:_) => () }
#[test] #[test]
fn test_rustc_issue_57597() { fn test_rustc_issue_57597() {
// <https://github.com/rust-lang/rust/blob/master/src/test/ui/macros/issue-57597.rs> // <https://github.com/rust-lang/rust/blob/master/src/test/ui/issues/issue-57597.rs>
check( check(
r#" r#"
macro_rules! m0 { ($($($i:ident)?)+) => {}; } macro_rules! m0 { ($($($i:ident)?)+) => {}; }

59
crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs
View file

@ -38,7 +38,6 @@ macro_rules! m {
let _ = 12E+99_f64; let _ = 12E+99_f64;
let _ = "rust1"; let _ = "rust1";
let _ = -92; let _ = -92;
let _ = -1.3e4f32;
} }
} }
fn f() { fn f() {
@ -53,7 +52,6 @@ macro_rules! m {
let _ = 12E+99_f64; let _ = 12E+99_f64;
let _ = "rust1"; let _ = "rust1";
let _ = -92; let _ = -92;
let _ = -1.3e4f32;
} }
} }
fn f() { fn f() {
@ -62,7 +60,6 @@ fn f() {
let _ = 12E+99_f64; let _ = 12E+99_f64;
let _ = "rust1"; let _ = "rust1";
let _ = -92; let _ = -92;
let _ = -1.3e4f32;
} }
"#]], "#]],
); );
@ -152,6 +149,32 @@ $ = ();
) )
} }
#[test]
fn float_literal_in_tt() {
check(
r#"
macro_rules! constant {
($( $ret:expr; )*) => {};
}
macro_rules! float_const_impl {
() => ( constant!(0.3; 3.3;); );
}
float_const_impl! {}
"#,
expect![[r#"
macro_rules! constant {
($( $ret:expr; )*) => {};
}
macro_rules! float_const_impl {
() => ( constant!(0.3; 3.3;); );
}
constant!(0.3;
3.3;
);
"#]],
);
}
#[test] #[test]
fn float_literal_in_output() { fn float_literal_in_output() {
check( check(
@ -175,33 +198,3 @@ const _: () = 0e0;
"#]], "#]],
); );
} }
#[test]
fn float_literal_in_tt() {
check(
r#"
macro_rules! constant {
($( $ret:expr; )*) => {};
}
macro_rules! float_const_impl {
() => ( constant!(0.3; 3.3;); );
}
float_const_impl! {}
"#,
expect![[r#"
macro_rules! constant {
($( $ret:expr; )*) => {};
}
macro_rules! float_const_impl {
() => ( constant!(0.3; 3.3;); );
}
constant!(0.3;
3.3;
);
"#]],
);
}

5
crates/hir-def/src/macro_expansion_tests/proc_macros.rs
View file

@ -104,8 +104,7 @@ macro_rules! id {
$($t)* $($t)*
}; };
} }
id /*+errors*/! {
id! {
#[proc_macros::identity] #[proc_macros::identity]
impl Foo for WrapBj { impl Foo for WrapBj {
async fn foo(&self) { async fn foo(&self) {
@ -120,7 +119,7 @@ macro_rules! id {
$($t)* $($t)*
}; };
} }
/* parse error: expected SEMICOLON */
#[proc_macros::identity] impl Foo for WrapBj { #[proc_macros::identity] impl Foo for WrapBj {
async fn foo(&self ) { async fn foo(&self ) {
self .0.id().await ; self .0.id().await ;

36
crates/hir-expand/src/builtin_fn_macro.rs
View file

@ -4,7 +4,10 @@ use base_db::{AnchoredPath, Edition, FileId};
use cfg::CfgExpr; use cfg::CfgExpr;
use either::Either; use either::Either;
use mbe::{parse_exprs_with_sep, parse_to_token_tree}; use mbe::{parse_exprs_with_sep, parse_to_token_tree};
use syntax::{ast, SmolStr}; use syntax::{
ast::{self, AstToken},
SmolStr,
};
use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc}; use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc};
@ -355,7 +358,14 @@ fn unreachable_expand(
} }
fn unquote_str(lit: &tt::Literal) -> Option<String> { fn unquote_str(lit: &tt::Literal) -> Option<String> {
let token = ast::make::literal(&lit.to_string()).as_string()?; let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::String::cast(lit)?;
token.value().map(|it| it.into_owned())
}
fn unquote_byte_string(lit: &tt::Literal) -> Option<Vec<u8>> {
let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::ByteString::cast(lit)?;
token.value().map(|it| it.into_owned()) token.value().map(|it| it.into_owned())
} }
@ -432,16 +442,12 @@ fn concat_bytes_expand(
for (i, t) in tt.token_trees.iter().enumerate() { for (i, t) in tt.token_trees.iter().enumerate() {
match t { match t {
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
let lit = ast::make::literal(&lit.to_string()); let token = ast::make::tokens::literal(&lit.to_string());
match lit.kind() { match token.kind() {
ast::LiteralKind::ByteString(s) => { syntax::SyntaxKind::BYTE => bytes.push(token.text().to_string()),
s.value() syntax::SyntaxKind::BYTE_STRING => {
.unwrap_or_default() let components = unquote_byte_string(lit).unwrap_or_else(Vec::new);
.into_iter() components.into_iter().for_each(|x| bytes.push(x.to_string()));
.for_each(|x| bytes.push(x.to_string()));
}
ast::LiteralKind::Byte(_) => {
bytes.push(lit.to_string());
} }
_ => { _ => {
err.get_or_insert(mbe::ExpandError::UnexpectedToken.into()); err.get_or_insert(mbe::ExpandError::UnexpectedToken.into());
@ -475,10 +481,10 @@ fn concat_bytes_expand_subtree(
for (ti, tt) in tree.token_trees.iter().enumerate() { for (ti, tt) in tree.token_trees.iter().enumerate() {
match tt { match tt {
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
let lit = ast::make::literal(&lit.to_string()); let lit = ast::make::tokens::literal(&lit.to_string());
match lit.kind() { match lit.kind() {
ast::LiteralKind::IntNumber(_) | ast::LiteralKind::Byte(_) => { syntax::SyntaxKind::BYTE | syntax::SyntaxKind::INT_NUMBER => {
bytes.push(lit.to_string()); bytes.push(lit.text().to_string())
} }
_ => { _ => {
return Err(mbe::ExpandError::UnexpectedToken.into()); return Err(mbe::ExpandError::UnexpectedToken.into());

11
crates/hir-ty/src/tests/simple.rs
View file

@ -2733,14 +2733,3 @@ fn f() {
"#, "#,
); );
} }
#[test]
fn nested_tuple_index() {
check_no_mismatches(
r#"
fn main() {
let fld: i32 = ((0,),).0.0;
}
"#,
);
}

20
crates/ide-completion/src/completions/dot.rs
View file

@ -793,24 +793,4 @@ fn main() {
", ",
) )
} }
#[test]
fn tuple_index_completion() {
check(
r#"
struct I;
impl I {
fn i_method(&self) {}
}
struct S((), I);
fn f(s: S) {
s.1.$0
}
"#,
expect![[r#"
me i_method() fn(&self)
"#]],
);
}
} }

4
crates/ide-completion/src/completions/postfix.rs
View file

@ -5,7 +5,7 @@ mod format_like;
use hir::{Documentation, HasAttrs}; use hir::{Documentation, HasAttrs};
use ide_db::{imports::insert_use::ImportScope, ty_filter::TryEnum, SnippetCap}; use ide_db::{imports::insert_use::ImportScope, ty_filter::TryEnum, SnippetCap};
use syntax::{ use syntax::{
ast::{self, AstNode, LiteralKind}, ast::{self, AstNode, AstToken},
SyntaxKind::{EXPR_STMT, STMT_LIST}, SyntaxKind::{EXPR_STMT, STMT_LIST},
TextRange, TextSize, TextRange, TextSize,
}; };
@ -194,7 +194,7 @@ pub(crate) fn complete_postfix(acc: &mut Completions, ctx: &CompletionContext) {
} }
if let ast::Expr::Literal(literal) = dot_receiver.clone() { if let ast::Expr::Literal(literal) = dot_receiver.clone() {
if let LiteralKind::String(literal_text) = literal.kind() { if let Some(literal_text) = ast::String::cast(literal.token()) {
add_format_like_completions(acc, ctx, &dot_receiver, cap, &literal_text); add_format_like_completions(acc, ctx, &dot_receiver, cap, &literal_text);
} }
} }

2
crates/ide-completion/src/context.rs
View file

@ -1062,7 +1062,7 @@ impl<'a> CompletionContext<'a> {
let receiver_is_ambiguous_float_literal = match &receiver { let receiver_is_ambiguous_float_literal = match &receiver {
Some(ast::Expr::Literal(l)) => matches! { Some(ast::Expr::Literal(l)) => matches! {
l.kind(), l.kind(),
ast::LiteralKind::FloatNumber { .. } if l.syntax().last_token().map_or(false, |it| it.kind() == T![.]) ast::LiteralKind::FloatNumber { .. } if l.syntax().last_token().map_or(false, |it| it.text().ends_with('.'))
}, },
_ => false, _ => false,
}; };

10
crates/ide/src/syntax_highlighting/highlight.rs
View file

@ -30,15 +30,7 @@ pub(super) fn token(sema: &Semantics<RootDatabase>, token: SyntaxToken) -> Optio
INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => { INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => {
SymbolKind::Field.into() SymbolKind::Field.into()
} }
INT_NUMBER | FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 INT_NUMBER | FLOAT_NUMBER => HlTag::NumericLiteral.into(),
| FLOAT_NUMBER_START_2 => HlTag::NumericLiteral.into(),
DOT if matches!(
token.prev_token().map(|n| n.kind()),
Some(FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2)
) =>
{
HlTag::NumericLiteral.into()
}
BYTE => HlTag::ByteLiteral.into(), BYTE => HlTag::ByteLiteral.into(),
CHAR => HlTag::CharLiteral.into(), CHAR => HlTag::CharLiteral.into(),
IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => { IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => {

14
crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
View file

@ -119,13 +119,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">-</span><span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">-</span><span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">#</span><span class="numeric_literal">0</span><span class="numeric_literal">10</span><span class="variable">x</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">27</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">#</span><span class="numeric_literal">0</span><span class="numeric_literal">10</span><span class="variable">x</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">27</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234</span><span class="numeric_literal">.</span><span class="numeric_literal">56</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234.56</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">&gt;</span><span class="numeric_literal">8</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 right-aligned characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">&gt;</span><span class="numeric_literal">8</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 right-aligned characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello {{}}"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello {{}}"</span><span class="parenthesis">)</span><span class="semicolon">;</span>

79
crates/mbe/src/syntax_bridge.rs
View file

@ -243,8 +243,6 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree {
let char = match token.to_char(conv) { let char = match token.to_char(conv) {
Some(c) => c, Some(c) => c,
None => { None => {
// FIXME: this isn't really correct, `to_char` yields the *first* char of the token,
// and this is relevant when eg. creating 2 `tt::Punct` from a single `::` token
panic!("Token from lexer must be single char: token = {:#?}", token); panic!("Token from lexer must be single char: token = {:#?}", token);
} }
}; };
@ -262,33 +260,6 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree {
IDENT => make_leaf!(Ident), IDENT => make_leaf!(Ident),
UNDERSCORE => make_leaf!(Ident), UNDERSCORE => make_leaf!(Ident),
k if k.is_keyword() => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident),
FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => {
// Reassemble a split-up float token.
let mut range = range;
let mut text = token.to_text(conv).to_string();
if kind == FLOAT_NUMBER_START_1 || kind == FLOAT_NUMBER_START_2 {
let (dot, dot_range) = conv.bump().unwrap();
assert_eq!(dot.kind(conv), DOT);
text += &*dot.to_text(conv);
range = TextRange::new(range.start(), dot_range.end());
if kind == FLOAT_NUMBER_START_2 {
let (tail, tail_range) = conv.bump().unwrap();
assert_eq!(tail.kind(conv), FLOAT_NUMBER_PART);
text += &*tail.to_text(conv);
range = TextRange::new(range.start(), tail_range.end());
}
}
result.push(
tt::Leaf::from(tt::Literal {
id: conv.id_alloc().alloc(range, synth_id),
text: text.into(),
})
.into(),
);
continue;
}
k if k.is_literal() => make_leaf!(Literal), k if k.is_literal() => make_leaf!(Literal),
LIFETIME_IDENT => { LIFETIME_IDENT => {
let char_unit = TextSize::of('\''); let char_unit = TextSize::of('\'');
@ -742,7 +713,6 @@ struct TtTreeSink<'a> {
text_pos: TextSize, text_pos: TextSize,
inner: SyntaxTreeBuilder, inner: SyntaxTreeBuilder,
token_map: TokenMap, token_map: TokenMap,
remaining_float_lit_text: String,
} }
impl<'a> TtTreeSink<'a> { impl<'a> TtTreeSink<'a> {
@ -754,7 +724,6 @@ impl<'a> TtTreeSink<'a> {
text_pos: 0.into(), text_pos: 0.into(),
inner: SyntaxTreeBuilder::default(), inner: SyntaxTreeBuilder::default(),
token_map: TokenMap::default(), token_map: TokenMap::default(),
remaining_float_lit_text: String::new(),
} }
} }
@ -781,54 +750,6 @@ impl<'a> TtTreeSink<'a> {
n_tokens = 2; n_tokens = 2;
} }
// We need to split a float `tt::Literal` into up to 3 tokens consumed by the parser.
match self.cursor.token_tree() {
Some(tt::buffer::TokenTreeRef::Subtree(sub, _)) if sub.delimiter.is_none() => {
self.cursor = self.cursor.subtree().unwrap()
}
_ => {}
}
let literal = match self.cursor.token_tree() {
Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Literal(lit), _)) => Some(lit),
_ => None,
};
if matches!(
kind,
FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2
) {
if self.remaining_float_lit_text.is_empty() {
always!(
literal.is_some(),
"kind={:?}, cursor tt={:?}",
kind,
self.cursor.token_tree()
);
let text = literal.map_or(String::new(), |lit| lit.to_string());
self.cursor = self.cursor.bump();
match text.split_once('.') {
Some((start, end)) => {
self.inner.token(kind, start);
self.remaining_float_lit_text = format!(".{end}");
return;
}
None => {
self.inner.token(kind, &text);
return;
}
}
} else {
self.inner.token(kind, &self.remaining_float_lit_text);
self.remaining_float_lit_text.clear();
return;
}
}
if kind == DOT && !self.remaining_float_lit_text.is_empty() {
always!(self.remaining_float_lit_text.chars().next() == Some('.'));
self.inner.token(kind, ".");
self.remaining_float_lit_text = self.remaining_float_lit_text[1..].to_string();
return;
}
let mut last = self.cursor; let mut last = self.cursor;
for _ in 0..n_tokens { for _ in 0..n_tokens {
let tmp: u8; let tmp: u8;

16
crates/mbe/src/to_parser_input.rs
View file

@ -35,13 +35,15 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input {
let is_negated = lit.text.starts_with('-'); let is_negated = lit.text.starts_with('-');
let inner_text = &lit.text[if is_negated { 1 } else { 0 }..]; let inner_text = &lit.text[if is_negated { 1 } else { 0 }..];
let lexed_str = parser::LexedStr::new(inner_text); let kind = parser::LexedStr::single_token(inner_text)
if lexed_str.is_empty() { .map(|(kind, _error)| kind)
panic!("failed to convert literal: {:?}", lit); .filter(|kind| {
} kind.is_literal()
for i in 0..lexed_str.len() { && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER))
res.push(lexed_str.kind(i)); })
} .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit));
res.push(kind);
} }
tt::Leaf::Ident(ident) => match ident.text.as_ref() { tt::Leaf::Ident(ident) => match ident.text.as_ref() {
"_" => res.push(T![_]), "_" => res.push(T![_]),

11
crates/mbe/src/tt_iter.rs
View file

@ -90,20 +90,9 @@ impl<'a> TtIter<'a> {
let mut cursor = buffer.begin(); let mut cursor = buffer.begin();
let mut error = false; let mut error = false;
let mut float_fragments_to_skip = 0;
for step in tree_traversal.iter() { for step in tree_traversal.iter() {
match step { match step {
parser::Step::Token { kind, mut n_input_tokens } => { parser::Step::Token { kind, mut n_input_tokens } => {
if float_fragments_to_skip > 0 {
float_fragments_to_skip -= 1;
n_input_tokens = 0;
}
match kind {
SyntaxKind::LIFETIME_IDENT => n_input_tokens = 2,
SyntaxKind::FLOAT_NUMBER_START_1 => float_fragments_to_skip = 1,
SyntaxKind::FLOAT_NUMBER_START_2 => float_fragments_to_skip = 2,
_ => {}
}
if kind == SyntaxKind::LIFETIME_IDENT { if kind == SyntaxKind::LIFETIME_IDENT {
n_input_tokens = 2; n_input_tokens = 2;
} }

13
crates/parser/src/grammar.rs
View file

@ -39,7 +39,6 @@ mod generic_params;
mod types; mod types;
use crate::{ use crate::{
grammar::expressions::FLOAT_LITERAL_FIRST,
parser::{CompletedMarker, Marker, Parser}, parser::{CompletedMarker, Marker, Parser},
SyntaxKind::{self, *}, SyntaxKind::{self, *},
TokenSet, T, TokenSet, T,
@ -319,17 +318,9 @@ fn name_ref(p: &mut Parser) {
} }
fn name_ref_or_index(p: &mut Parser) { fn name_ref_or_index(p: &mut Parser) {
assert!( assert!(p.at(IDENT) || p.at(INT_NUMBER));
p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST)
);
let m = p.start(); let m = p.start();
if p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) { p.bump_any();
// Ideally we'd remap this to `INT_NUMBER` instead, but that causes the MBE conversion to
// lose track of what's a float and what isn't, causing panics.
p.bump_remap(FLOAT_NUMBER_PART);
} else {
p.bump_any();
}
m.complete(p, NAME_REF); m.complete(p, NAME_REF);
} }

12
crates/parser/src/grammar/expressions.rs
View file

@ -3,7 +3,7 @@ mod atom;
use super::*; use super::*;
pub(crate) use self::atom::{block_expr, match_arm_list}; pub(crate) use self::atom::{block_expr, match_arm_list};
pub(super) use self::atom::{literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST}; pub(super) use self::atom::{literal, LITERAL_FIRST};
#[derive(PartialEq, Eq)] #[derive(PartialEq, Eq)]
pub(super) enum Semicolon { pub(super) enum Semicolon {
@ -452,9 +452,6 @@ fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
// fn foo() { // fn foo() {
// x.foo(); // x.foo();
// y.bar::<T>(1, 2,); // y.bar::<T>(1, 2,);
//
// 0e0.sin();
// 0e0f32.sin();
// } // }
fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::]))); assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
@ -472,16 +469,17 @@ fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
// fn foo() { // fn foo() {
// x.foo; // x.foo;
// x.0.bar; // x.0.bar;
// x.0. bar;
// x.0.1;
// x.0(); // x.0();
// } // }
fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.])); assert!(p.at(T![.]));
let m = lhs.precede(p); let m = lhs.precede(p);
p.bump(T![.]); p.bump(T![.]);
if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) { if p.at(IDENT) || p.at(INT_NUMBER) {
name_ref_or_index(p); name_ref_or_index(p);
} else if p.at(FLOAT_NUMBER) {
// FIXME: How to recover and instead parse INT + T![.]?
p.bump_any();
} else { } else {
p.error("expected field name or number"); p.error("expected field name or number");
} }

40
crates/parser/src/grammar/expressions/atom.rs
View file

@ -17,58 +17,22 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[
T![true], T![true],
T![false], T![false],
INT_NUMBER, INT_NUMBER,
FLOAT_NUMBER_START_0, FLOAT_NUMBER,
FLOAT_NUMBER_START_1,
FLOAT_NUMBER_START_2,
BYTE, BYTE,
CHAR, CHAR,
STRING, STRING,
BYTE_STRING, BYTE_STRING,
]); ]);
pub(crate) const FLOAT_LITERAL_FIRST: TokenSet =
TokenSet::new(&[FLOAT_NUMBER_START_0, FLOAT_NUMBER_START_1, FLOAT_NUMBER_START_2]);
pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> { pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
if !p.at_ts(LITERAL_FIRST) { if !p.at_ts(LITERAL_FIRST) {
return None; return None;
} }
let m = p.start(); let m = p.start();
if p.at_ts(FLOAT_LITERAL_FIRST) { p.bump_any();
float_literal(p);
} else {
// Everything else is just one token.
p.bump_any();
}
Some(m.complete(p, LITERAL)) Some(m.complete(p, LITERAL))
} }
// test float_literal
// fn f() {
// 0.0;
// 1.;
// 0e0;
// 0e0f32;
// 1.23f64;
// }
pub(crate) fn float_literal(p: &mut Parser) {
// Floats can be up to 3 tokens. The first token indicates how many there are.
let f = p.start();
if p.at(FLOAT_NUMBER_START_0) {
p.bump(FLOAT_NUMBER_START_0);
} else if p.at(FLOAT_NUMBER_START_1) {
p.bump(FLOAT_NUMBER_START_1);
p.bump(DOT);
} else if p.at(FLOAT_NUMBER_START_2) {
p.bump(FLOAT_NUMBER_START_2);
p.bump(DOT);
p.bump(FLOAT_NUMBER_PART);
} else {
unreachable!();
}
f.complete(p, FLOAT_LITERAL);
}
// E.g. for after the break in `if break {}`, this should not match // E.g. for after the break in `if break {}`, this should not match
pub(super) const ATOM_EXPR_FIRST: TokenSet = pub(super) const ATOM_EXPR_FIRST: TokenSet =
LITERAL_FIRST.union(paths::PATH_FIRST).union(TokenSet::new(&[ LITERAL_FIRST.union(paths::PATH_FIRST).union(TokenSet::new(&[

2
crates/parser/src/grammar/patterns.rs
View file

@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> {
} }
fn is_literal_pat_start(p: &Parser) -> bool { fn is_literal_pat_start(p: &Parser) -> bool {
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART) p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER)
|| p.at_ts(expressions::LITERAL_FIRST) || p.at_ts(expressions::LITERAL_FIRST)
} }

28
crates/parser/src/lexed_str.rs
View file

@ -177,7 +177,7 @@ impl<'a> Converter<'a> {
rustc_lexer::TokenKind::RawIdent => IDENT, rustc_lexer::TokenKind::RawIdent => IDENT,
rustc_lexer::TokenKind::Literal { kind, .. } => { rustc_lexer::TokenKind::Literal { kind, .. } => {
self.extend_literal(token_text, kind); self.extend_literal(token_text.len(), kind);
return; return;
} }
@ -223,7 +223,7 @@ impl<'a> Converter<'a> {
self.push(syntax_kind, token_text.len(), err); self.push(syntax_kind, token_text.len(), err);
} }
fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) { fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) {
let mut err = ""; let mut err = "";
let syntax_kind = match *kind { let syntax_kind = match *kind {
@ -237,27 +237,7 @@ impl<'a> Converter<'a> {
if empty_exponent { if empty_exponent {
err = "Missing digits after the exponent symbol"; err = "Missing digits after the exponent symbol";
} }
FLOAT_NUMBER
// In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0`
// is lexed as a float, we split floats that contain a `.` into 3 tokens.
// To ensure that later stages can always reconstruct the token correctly, the first
// token in the sequence indicates the number of following tokens that are part of
// the float literal.
if let Some((before, after)) = token_text.split_once('.') {
let err = if err.is_empty() { None } else { Some(err) };
assert!(!before.is_empty());
let tok =
if after.is_empty() { FLOAT_NUMBER_START_1 } else { FLOAT_NUMBER_START_2 };
self.push(tok, before.len(), None);
self.push(DOT, 1, None);
if !after.is_empty() {
self.push(FLOAT_NUMBER_PART, after.len(), err);
}
return;
}
FLOAT_NUMBER_START_0
} }
rustc_lexer::LiteralKind::Char { terminated } => { rustc_lexer::LiteralKind::Char { terminated } => {
if !terminated { if !terminated {
@ -315,6 +295,6 @@ impl<'a> Converter<'a> {
}; };
let err = if err.is_empty() { None } else { Some(err) }; let err = if err.is_empty() { None } else { Some(err) };
self.push(syntax_kind, token_text.len(), err); self.push(syntax_kind, len, err);
} }
} }

11
crates/parser/src/syntax_kind/generated.rs
View file

File diff suppressed because one or more lines are too long

44
crates/parser/test_data/lexer/err/empty_exponent.rast
View file

@ -1,14 +1,14 @@
FLOAT_NUMBER_START_0 "0e" error: Missing digits after the exponent symbol FLOAT_NUMBER "0e" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_0 "0E" error: Missing digits after the exponent symbol FLOAT_NUMBER "0E" error: Missing digits after the exponent symbol
WHITESPACE "\n\n" WHITESPACE "\n\n"
FLOAT_NUMBER_START_0 "42e+" error: Missing digits after the exponent symbol FLOAT_NUMBER "42e+" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_0 "42e-" error: Missing digits after the exponent symbol FLOAT_NUMBER "42e-" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_0 "42E+" error: Missing digits after the exponent symbol FLOAT_NUMBER "42E+" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_0 "42E-" error: Missing digits after the exponent symbol FLOAT_NUMBER "42E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n" WHITESPACE "\n\n"
INT_NUMBER "42" INT_NUMBER "42"
DOT "." DOT "."
@ -30,35 +30,19 @@ DOT "."
IDENT "E" IDENT "E"
MINUS "-" MINUS "-"
WHITESPACE "\n\n" WHITESPACE "\n\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2e+" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2e-" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2E+" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2E-" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n" WHITESPACE "\n\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2e+f32" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2e-f32" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2E+f32" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2E-f32" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"

25
crates/parser/test_data/lexer/ok/numbers.rast
View file

@ -4,8 +4,7 @@ INT_NUMBER "00"
WHITESPACE " " WHITESPACE " "
INT_NUMBER "0_" INT_NUMBER "0_"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER_START_1 "0" FLOAT_NUMBER "0."
DOT "."
WHITESPACE " " WHITESPACE " "
INT_NUMBER "0z" INT_NUMBER "0z"
WHITESPACE "\n" WHITESPACE "\n"
@ -21,13 +20,11 @@ INT_NUMBER "001279"
WHITESPACE " " WHITESPACE " "
INT_NUMBER "0_1279" INT_NUMBER "0_1279"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER_START_2 "0" FLOAT_NUMBER "0.1279"
DOT "."
FLOAT_NUMBER_PART "1279"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER_START_0 "0e1279" FLOAT_NUMBER "0e1279"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER_START_0 "0E1279" FLOAT_NUMBER "0E1279"
WHITESPACE "\n" WHITESPACE "\n"
INT_NUMBER "0" INT_NUMBER "0"
DOT "." DOT "."
@ -40,7 +37,7 @@ IDENT "foo"
L_PAREN "(" L_PAREN "("
R_PAREN ")" R_PAREN ")"
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_0 "0e+1" FLOAT_NUMBER "0e+1"
WHITESPACE "\n" WHITESPACE "\n"
INT_NUMBER "0" INT_NUMBER "0"
DOT "." DOT "."
@ -48,19 +45,13 @@ IDENT "e"
PLUS "+" PLUS "+"
INT_NUMBER "1" INT_NUMBER "1"
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "0" FLOAT_NUMBER "0.0E-2"
DOT "."
FLOAT_NUMBER_PART "0E-2"
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "0___0" FLOAT_NUMBER "0___0.10000____0000e+111__"
DOT "."
FLOAT_NUMBER_PART "10000____0000e+111__"
WHITESPACE "\n" WHITESPACE "\n"
INT_NUMBER "1i64" INT_NUMBER "1i64"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER_START_2 "92" FLOAT_NUMBER "92.0f32"
DOT "."
FLOAT_NUMBER_PART "0f32"
WHITESPACE " " WHITESPACE " "
INT_NUMBER "11__s" INT_NUMBER "11__s"
WHITESPACE "\n" WHITESPACE "\n"

4
crates/parser/test_data/parser/err/0023_mismatched_paren.rast
View file

@ -32,9 +32,7 @@ SOURCE_FILE
INT_NUMBER "1" INT_NUMBER "1"
COMMA "," COMMA ","
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER_START_2 "2" FLOAT_NUMBER "2.0"
DOT "."
FLOAT_NUMBER_PART "0"
WHITESPACE "\n " WHITESPACE "\n "
R_CURLY "}" R_CURLY "}"
WHITESPACE " " WHITESPACE " "

33
crates/parser/test_data/parser/inline/ok/0011_field_expr.rast
View file

@ -40,39 +40,6 @@ SOURCE_FILE
IDENT "bar" IDENT "bar"
SEMICOLON ";" SEMICOLON ";"
WHITESPACE "\n " WHITESPACE "\n "
EXPR_STMT
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "0"
DOT "."
WHITESPACE " "
NAME_REF
IDENT "bar"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "0"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "1"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT EXPR_STMT
CALL_EXPR CALL_EXPR
FIELD_EXPR FIELD_EXPR

2
crates/parser/test_data/parser/inline/ok/0011_field_expr.rs
View file

@ -1,7 +1,5 @@
fn foo() { fn foo() {
x.foo; x.foo;
x.0.bar; x.0.bar;
x.0. bar;
x.0.1;
x.0(); x.0();
} }

5
crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast
View file

@ -57,10 +57,7 @@ SOURCE_FILE
EQ "=" EQ "="
WHITESPACE " " WHITESPACE " "
LITERAL LITERAL
FLOAT_LITERAL FLOAT_NUMBER "2.0"
FLOAT_NUMBER_START_2 "2"
DOT "."
FLOAT_NUMBER_PART "0"
SEMICOLON ";" SEMICOLON ";"
WHITESPACE "\n " WHITESPACE "\n "
LET_STMT LET_STMT

26
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast
View file

@ -58,32 +58,6 @@ SOURCE_FILE
COMMA "," COMMA ","
R_PAREN ")" R_PAREN ")"
SEMICOLON ";" SEMICOLON ";"
WHITESPACE "\n\n "
EXPR_STMT
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0"
DOT "."
NAME_REF
IDENT "sin"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0f32"
DOT "."
NAME_REF
IDENT "sin"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n" WHITESPACE "\n"
R_CURLY "}" R_CURLY "}"
WHITESPACE "\n" WHITESPACE "\n"

3
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs
View file

@ -1,7 +1,4 @@
fn foo() { fn foo() {
x.foo(); x.foo();
y.bar::<T>(1, 2,); y.bar::<T>(1, 2,);
0e0.sin();
0e0f32.sin();
} }

51
crates/parser/test_data/parser/inline/ok/0201_float_literal.rast
View file

@ -1,51 +0,0 @@
SOURCE_FILE
FN
FN_KW "fn"
WHITESPACE " "
NAME
IDENT "f"
PARAM_LIST
L_PAREN "("
R_PAREN ")"
WHITESPACE " "
BLOCK_EXPR
STMT_LIST
L_CURLY "{"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "0"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_1 "1"
DOT "."
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0f32"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "23f64"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"

7
crates/parser/test_data/parser/inline/ok/0201_float_literal.rs
View file

@ -1,7 +0,0 @@
fn f() {
0.0;
1.;
0e0;
0e0f32;
1.23f64;
}

10
crates/parser/test_data/parser/ok/0056_neq_in_type.rast
View file

@ -19,10 +19,7 @@ SOURCE_FILE
CAST_EXPR CAST_EXPR
METHOD_CALL_EXPR METHOD_CALL_EXPR
LITERAL LITERAL
FLOAT_LITERAL FLOAT_NUMBER "1.0f32"
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "0f32"
DOT "." DOT "."
NAME_REF NAME_REF
IDENT "floor" IDENT "floor"
@ -43,10 +40,7 @@ SOURCE_FILE
CAST_EXPR CAST_EXPR
METHOD_CALL_EXPR METHOD_CALL_EXPR
LITERAL LITERAL
FLOAT_LITERAL FLOAT_NUMBER "1.0f32"
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "0f32"
DOT "." DOT "."
NAME_REF NAME_REF
IDENT "floor" IDENT "floor"

9
crates/syntax/rust.ungram
View file

@ -365,20 +365,13 @@ MacroExpr =
Literal = Literal =
Attr* value:( Attr* value:(
'int_number' | FloatLiteral 'int_number' | 'float_number'
| 'string' | 'raw_string' | 'string' | 'raw_string'
| 'byte_string' | 'raw_byte_string' | 'byte_string' | 'raw_byte_string'
| 'true' | 'false' | 'true' | 'false'
| 'char' | 'byte' | 'char' | 'byte'
) )
FloatLiteral =
'float_number_start_0'?
'float_number_start_1'?
'float_number_start_2'?
'.'?
'float_number_part'?
PathExpr = PathExpr =
Attr* Path Attr* Path

42
crates/syntax/src/ast/expr_ext.rs
View file

@ -8,7 +8,7 @@ use crate::{
operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp}, operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp},
support, AstChildren, AstNode, support, AstChildren, AstNode,
}, },
AstToken, SyntaxElement, AstToken,
SyntaxKind::*, SyntaxKind::*,
SyntaxNode, SyntaxToken, T, SyntaxNode, SyntaxToken, T,
}; };
@ -282,32 +282,30 @@ pub enum LiteralKind {
String(ast::String), String(ast::String),
ByteString(ast::ByteString), ByteString(ast::ByteString),
IntNumber(ast::IntNumber), IntNumber(ast::IntNumber),
FloatNumber(ast::FloatLiteral), FloatNumber(ast::FloatNumber),
Char(ast::Char), Char(ast::Char),
Byte(ast::Byte), Byte(ast::Byte),
Bool(bool), Bool(bool),
} }
impl ast::Literal { impl ast::Literal {
pub fn value(&self) -> SyntaxElement { pub fn token(&self) -> SyntaxToken {
self.syntax() self.syntax()
.children_with_tokens() .children_with_tokens()
.find(|e| e.kind() != ATTR && !e.kind().is_trivia()) .find(|e| e.kind() != ATTR && !e.kind().is_trivia())
.and_then(|e| e.into_token())
.unwrap() .unwrap()
} }
pub fn kind(&self) -> LiteralKind { pub fn kind(&self) -> LiteralKind {
let token = match self.value() { let token = self.token();
rowan::NodeOrToken::Node(node) => {
return LiteralKind::FloatNumber(
ast::FloatLiteral::cast(node).expect("unreachable"),
);
}
rowan::NodeOrToken::Token(token) => token,
};
if let Some(t) = ast::IntNumber::cast(token.clone()) { if let Some(t) = ast::IntNumber::cast(token.clone()) {
return LiteralKind::IntNumber(t); return LiteralKind::IntNumber(t);
} }
if let Some(t) = ast::FloatNumber::cast(token.clone()) {
return LiteralKind::FloatNumber(t);
}
if let Some(t) = ast::String::cast(token.clone()) { if let Some(t) = ast::String::cast(token.clone()) {
return LiteralKind::String(t); return LiteralKind::String(t);
} }
@ -327,26 +325,6 @@ impl ast::Literal {
_ => unreachable!(), _ => unreachable!(),
} }
} }
pub fn as_string(&self) -> Option<ast::String> {
match self.kind() {
LiteralKind::String(it) => Some(it),
_ => None,
}
}
pub fn as_byte_string(&self) -> Option<ast::ByteString> {
match self.kind() {
LiteralKind::ByteString(it) => Some(it),
_ => None,
}
}
}
impl ast::FloatLiteral {
pub fn suffix(&self) -> Option<String> {
ast::FloatNumberPart::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string())
}
} }
pub enum BlockModifier { pub enum BlockModifier {
@ -386,7 +364,7 @@ impl ast::BlockExpr {
fn test_literal_with_attr() { fn test_literal_with_attr() {
let parse = ast::SourceFile::parse(r#"const _: &str = { #[attr] "Hello" };"#); let parse = ast::SourceFile::parse(r#"const _: &str = { #[attr] "Hello" };"#);
let lit = parse.tree().syntax().descendants().find_map(ast::Literal::cast).unwrap(); let lit = parse.tree().syntax().descendants().find_map(ast::Literal::cast).unwrap();
assert_eq!(lit.value().to_string(), r#""Hello""#); assert_eq!(lit.token().text(), r#""Hello""#);
} }
impl ast::RecordExprField { impl ast::RecordExprField {

36
crates/syntax/src/ast/generated/nodes.rs
View file

@ -1085,26 +1085,6 @@ impl UnderscoreExpr {
pub fn underscore_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![_]) } pub fn underscore_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![_]) }
} }
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct FloatLiteral {
pub(crate) syntax: SyntaxNode,
}
impl FloatLiteral {
pub fn float_number_start_0_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_0])
}
pub fn float_number_start_1_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_1])
}
pub fn float_number_start_2_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_2])
}
pub fn dot_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![.]) }
pub fn float_number_part_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_part])
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct StmtList { pub struct StmtList {
pub(crate) syntax: SyntaxNode, pub(crate) syntax: SyntaxNode,
@ -2739,17 +2719,6 @@ impl AstNode for UnderscoreExpr {
} }
fn syntax(&self) -> &SyntaxNode { &self.syntax } fn syntax(&self) -> &SyntaxNode { &self.syntax }
} }
impl AstNode for FloatLiteral {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_LITERAL }
fn cast(syntax: SyntaxNode) -> Option<Self> {
if Self::can_cast(syntax.kind()) {
Some(Self { syntax })
} else {
None
}
}
fn syntax(&self) -> &SyntaxNode { &self.syntax }
}
impl AstNode for StmtList { impl AstNode for StmtList {
fn can_cast(kind: SyntaxKind) -> bool { kind == STMT_LIST } fn can_cast(kind: SyntaxKind) -> bool { kind == STMT_LIST }
fn cast(syntax: SyntaxNode) -> Option<Self> { fn cast(syntax: SyntaxNode) -> Option<Self> {
@ -4639,11 +4608,6 @@ impl std::fmt::Display for UnderscoreExpr {
std::fmt::Display::fmt(self.syntax(), f) std::fmt::Display::fmt(self.syntax(), f)
} }
} }
impl std::fmt::Display for FloatLiteral {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f)
}
}
impl std::fmt::Display for StmtList { impl std::fmt::Display for StmtList {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f) std::fmt::Display::fmt(self.syntax(), f)

8
crates/syntax/src/ast/generated/tokens.rs
View file

@ -112,16 +112,16 @@ impl AstToken for IntNumber {
} }
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct FloatNumberPart { pub struct FloatNumber {
pub(crate) syntax: SyntaxToken, pub(crate) syntax: SyntaxToken,
} }
impl std::fmt::Display for FloatNumberPart { impl std::fmt::Display for FloatNumber {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.syntax, f) std::fmt::Display::fmt(&self.syntax, f)
} }
} }
impl AstToken for FloatNumberPart { impl AstToken for FloatNumber {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER_PART } fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER }
fn cast(syntax: SyntaxToken) -> Option<Self> { fn cast(syntax: SyntaxToken) -> Option<Self> {
if Self::can_cast(syntax.kind()) { if Self::can_cast(syntax.kind()) {
Some(Self { syntax }) Some(Self { syntax })

13
crates/syntax/src/ast/make.rs
View file

@ -799,11 +799,6 @@ pub fn struct_(
)) ))
} }
pub fn literal(text: &str) -> ast::Literal {
assert_eq!(text.trim(), text);
ast_from_text(&format!("fn f() {{ let _ = {}; }}", text))
}
#[track_caller] #[track_caller]
fn ast_from_text<N: AstNode>(text: &str) -> N { fn ast_from_text<N: AstNode>(text: &str) -> N {
let parse = SourceFile::parse(text); let parse = SourceFile::parse(text);
@ -832,7 +827,7 @@ pub fn token(kind: SyntaxKind) -> SyntaxToken {
pub mod tokens { pub mod tokens {
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use crate::{AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken}; use crate::{ast, AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken};
pub(super) static SOURCE_FILE: Lazy<Parse<SourceFile>> = Lazy::new(|| { pub(super) static SOURCE_FILE: Lazy<Parse<SourceFile>> = Lazy::new(|| {
SourceFile::parse( SourceFile::parse(
@ -863,6 +858,12 @@ pub mod tokens {
sf.syntax().first_child_or_token().unwrap().into_token().unwrap() sf.syntax().first_child_or_token().unwrap().into_token().unwrap()
} }
pub fn literal(text: &str) -> SyntaxToken {
assert_eq!(text.trim(), text);
let lit: ast::Literal = super::ast_from_text(&format!("fn f() {{ let _ = {}; }}", text));
lit.syntax().first_child_or_token().unwrap().into_token().unwrap()
}
pub fn single_newline() -> SyntaxToken { pub fn single_newline() -> SyntaxToken {
let res = SOURCE_FILE let res = SOURCE_FILE
.tree() .tree()

4
crates/syntax/src/ast/node_ext.rs
View file

@ -555,9 +555,7 @@ impl ast::FieldExpr {
self.syntax self.syntax
.children_with_tokens() .children_with_tokens()
// FIXME: Accepting floats here to reject them in validation later // FIXME: Accepting floats here to reject them in validation later
.find(|c| { .find(|c| c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER)
c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER_PART
})
.as_ref() .as_ref()
.and_then(SyntaxElement::as_token) .and_then(SyntaxElement::as_token)
.cloned() .cloned()

29
crates/syntax/src/ast/token_ext.rs
View file

@ -321,7 +321,7 @@ impl ast::IntNumber {
} }
} }
impl ast::FloatNumberPart { impl ast::FloatNumber {
pub fn suffix(&self) -> Option<&str> { pub fn suffix(&self) -> Option<&str> {
let text = self.text(); let text = self.text();
let mut indices = text.char_indices(); let mut indices = text.char_indices();
@ -355,24 +355,14 @@ impl Radix {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::ast::{self, make}; use crate::ast::{self, make, FloatNumber, IntNumber};
fn check_float_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) { fn check_float_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
let suffix = match make::literal(lit).kind() { assert_eq!(FloatNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into());
ast::LiteralKind::FloatNumber(f) => f.suffix(),
// `1f32` lexes as an INT_NUMBER
ast::LiteralKind::IntNumber(i) => i.suffix().map(|s| s.to_string()),
e => unreachable!("{e:?}"),
};
assert_eq!(suffix.as_deref(), expected.into());
} }
fn check_int_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) { fn check_int_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
let i = match make::literal(lit).kind() { assert_eq!(IntNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into());
ast::LiteralKind::IntNumber(i) => i,
_ => unreachable!(),
};
assert_eq!(i.suffix(), expected.into());
} }
#[test] #[test]
@ -400,11 +390,12 @@ mod tests {
} }
fn check_string_value<'a>(lit: &str, expected: impl Into<Option<&'a str>>) { fn check_string_value<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
let s = match make::literal(&format!("\"{}\"", lit)).kind() { assert_eq!(
ast::LiteralKind::String(s) => s, ast::String { syntax: make::tokens::literal(&format!("\"{}\"", lit)) }
_ => unreachable!(), .value()
}; .as_deref(),
assert_eq!(s.value().as_deref(), expected.into()); expected.into()
);
} }
#[test] #[test]

13
crates/syntax/src/tests/ast_src.rs
View file

@ -71,17 +71,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
"super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield", "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield",
], ],
contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"], contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"],
literals: &[ literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"],
"INT_NUMBER",
"FLOAT_NUMBER_START_0",
"FLOAT_NUMBER_START_1",
"FLOAT_NUMBER_START_2",
"FLOAT_NUMBER_PART",
"CHAR",
"BYTE",
"STRING",
"BYTE_STRING",
],
tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"], tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"],
nodes: &[ nodes: &[
"SOURCE_FILE", "SOURCE_FILE",
@ -193,7 +183,6 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
"PATH", "PATH",
"PATH_SEGMENT", "PATH_SEGMENT",
"LITERAL", "LITERAL",
"FLOAT_LITERAL",
"RENAME", "RENAME",
"VISIBILITY", "VISIBILITY",
"WHERE_CLAUSE", "WHERE_CLAUSE",

6
crates/syntax/src/tests/sourcegen_ast.rs
View file

@ -462,10 +462,6 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String {
[lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT }; [lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT };
[ident] => { $crate::SyntaxKind::IDENT }; [ident] => { $crate::SyntaxKind::IDENT };
[shebang] => { $crate::SyntaxKind::SHEBANG }; [shebang] => { $crate::SyntaxKind::SHEBANG };
[float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART };
[float_number_start_0] => { $crate::SyntaxKind::FLOAT_NUMBER_START_0 };
[float_number_start_1] => { $crate::SyntaxKind::FLOAT_NUMBER_START_1 };
[float_number_start_2] => { $crate::SyntaxKind::FLOAT_NUMBER_START_2 };
} }
pub use T; pub use T;
}; };
@ -589,7 +585,7 @@ impl Field {
fn lower(grammar: &Grammar) -> AstSrc { fn lower(grammar: &Grammar) -> AstSrc {
let mut res = AstSrc { let mut res = AstSrc {
tokens: "Whitespace Comment String ByteString IntNumber FloatNumberPart Char Byte Ident" tokens: "Whitespace Comment String ByteString IntNumber FloatNumber Char Byte Ident"
.split_ascii_whitespace() .split_ascii_whitespace()
.map(|it| it.to_string()) .map(|it| it.to_string())
.collect::<Vec<_>>(), .collect::<Vec<_>>(),

11
crates/syntax/src/validation.rs
View file

@ -119,15 +119,8 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
text.rfind(end_delimiter).and_then(|end| text.get(prefix_len..end)) text.rfind(end_delimiter).and_then(|end| text.get(prefix_len..end))
} }
let token = literal.value(); let token = literal.token();
let text; let text = token.text();
let text = match &token {
rowan::NodeOrToken::Node(node) => {
text = node.text().to_string();
&*text
}
rowan::NodeOrToken::Token(token) => token.text(),
};
// FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205) // FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205)
let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| { let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| {