Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2025-01-13 21:54:42 +00:00
Code Issues Projects Releases Packages Wiki Activity

Auto merge of #12241 - jonas-schievink:does-not-float-my-boat, r=jonas-schievink

Browse source 
fix: revert float parsing "fix" to avoid macro-related panics

Reverts https://github.com/rust-lang/rust-analyzer/pull/12149 and the follow-up fixes, while keeping their tests.

https://github.com/rust-lang/rust-analyzer/pull/12149 has caused many unexpected panics related to macros, and the fixes for those are not straightforward and further complicate the MBE token conversion logic, which was already fairly hard to follow before these fixes.
This commit is contained in:
bors 2022-05-13 15:02:00 +00:00
commit 4f6b2a20fd
42 changed files with 156 additions and 610 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
crates/hir-def/src/body/lower.rs
View file

@ -972,7 +972,7 @@ impl From<ast::LiteralKind> for Literal {
}
}
LiteralKind::FloatNumber(lit) => {
let ty = lit.suffix().and_then(|s| BuiltinFloat::from_suffix(&s));
let ty = lit.suffix().and_then(BuiltinFloat::from_suffix);
Literal::Float(Default::default(), ty)
}
LiteralKind::ByteString(bs) => {

8
crates/hir-def/src/macro_expansion_tests/mbe.rs
View file

@ -48,6 +48,8 @@ struct#10 MyTraitMap2#32 {#13
#[test]
fn token_mapping_floats() {
// Regression test for https://github.com/rust-lang/rust-analyzer/issues/12216
// (and related issues)
check(
r#"
// +tokenids
@ -87,9 +89,9 @@ macro_rules! f {#0
// }
fn#19 main#20(#21)#21 {#22
1#23;#24
1#26.0;
let x#31 =#22 1;
}
1.0#25;#26
let#27 x#28 =#29 1#30;#31
}#22
"##]],

2
crates/hir-def/src/macro_expansion_tests/mbe/meta_syntax.rs
View file

@ -80,7 +80,7 @@ macro_rules! f3 { ($i:_) => () }
#[test]
fn test_rustc_issue_57597() {
// <https://github.com/rust-lang/rust/blob/master/src/test/ui/macros/issue-57597.rs>
// <https://github.com/rust-lang/rust/blob/master/src/test/ui/issues/issue-57597.rs>
check(
r#"
macro_rules! m0 { ($($($i:ident)?)+) => {}; }

59
crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs
View file

@ -38,7 +38,6 @@ macro_rules! m {
let _ = 12E+99_f64;
let _ = "rust1";
let _ = -92;
let _ = -1.3e4f32;
}
}
fn f() {
@ -53,7 +52,6 @@ macro_rules! m {
let _ = 12E+99_f64;
let _ = "rust1";
let _ = -92;
let _ = -1.3e4f32;
}
}
fn f() {
@ -62,7 +60,6 @@ fn f() {
let _ = 12E+99_f64;
let _ = "rust1";
let _ = -92;
let _ = -1.3e4f32;
}
"#]],
);
@ -152,6 +149,32 @@ $ = ();
)
}
#[test]
fn float_literal_in_tt() {
check(
r#"
macro_rules! constant {
($( $ret:expr; )*) => {};
}
macro_rules! float_const_impl {
() => ( constant!(0.3; 3.3;); );
}
float_const_impl! {}
"#,
expect![[r#"
macro_rules! constant {
($( $ret:expr; )*) => {};
}
macro_rules! float_const_impl {
() => ( constant!(0.3; 3.3;); );
}
constant!(0.3;
3.3;
);
"#]],
);
}
#[test]
fn float_literal_in_output() {
check(
@ -175,33 +198,3 @@ const _: () = 0e0;
"#]],
);
}
#[test]
fn float_literal_in_tt() {
check(
r#"
macro_rules! constant {
($( $ret:expr; )*) => {};
}
macro_rules! float_const_impl {
() => ( constant!(0.3; 3.3;); );
}
float_const_impl! {}
"#,
expect![[r#"
macro_rules! constant {
($( $ret:expr; )*) => {};
}
macro_rules! float_const_impl {
() => ( constant!(0.3; 3.3;); );
}
constant!(0.3;
3.3;
);
"#]],
);
}

5
crates/hir-def/src/macro_expansion_tests/proc_macros.rs
View file

@ -104,8 +104,7 @@ macro_rules! id {
$($t)*
};
}
id! {
id /*+errors*/! {
#[proc_macros::identity]
impl Foo for WrapBj {
async fn foo(&self) {
@ -120,7 +119,7 @@ macro_rules! id {
$($t)*
};
}
/* parse error: expected SEMICOLON */
#[proc_macros::identity] impl Foo for WrapBj {
async fn foo(&self ) {
self .0.id().await ;

36
crates/hir-expand/src/builtin_fn_macro.rs
View file

@ -4,7 +4,10 @@ use base_db::{AnchoredPath, Edition, FileId};
use cfg::CfgExpr;
use either::Either;
use mbe::{parse_exprs_with_sep, parse_to_token_tree};
use syntax::{ast, SmolStr};
use syntax::{
ast::{self, AstToken},
SmolStr,
};
use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc};
@ -355,7 +358,14 @@ fn unreachable_expand(
}
fn unquote_str(lit: &tt::Literal) -> Option<String> {
let token = ast::make::literal(&lit.to_string()).as_string()?;
let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::String::cast(lit)?;
token.value().map(|it| it.into_owned())
}
fn unquote_byte_string(lit: &tt::Literal) -> Option<Vec<u8>> {
let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::ByteString::cast(lit)?;
token.value().map(|it| it.into_owned())
}
@ -432,16 +442,12 @@ fn concat_bytes_expand(
for (i, t) in tt.token_trees.iter().enumerate() {
match t {
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
let lit = ast::make::literal(&lit.to_string());
match lit.kind() {
ast::LiteralKind::ByteString(s) => {
s.value()
.unwrap_or_default()
.into_iter()
.for_each(|x| bytes.push(x.to_string()));
}
ast::LiteralKind::Byte(_) => {
bytes.push(lit.to_string());
let token = ast::make::tokens::literal(&lit.to_string());
match token.kind() {
syntax::SyntaxKind::BYTE => bytes.push(token.text().to_string()),
syntax::SyntaxKind::BYTE_STRING => {
let components = unquote_byte_string(lit).unwrap_or_else(Vec::new);
components.into_iter().for_each(|x| bytes.push(x.to_string()));
}
_ => {
err.get_or_insert(mbe::ExpandError::UnexpectedToken.into());
@ -475,10 +481,10 @@ fn concat_bytes_expand_subtree(
for (ti, tt) in tree.token_trees.iter().enumerate() {
match tt {
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
let lit = ast::make::literal(&lit.to_string());
let lit = ast::make::tokens::literal(&lit.to_string());
match lit.kind() {
ast::LiteralKind::IntNumber(_) | ast::LiteralKind::Byte(_) => {
bytes.push(lit.to_string());
syntax::SyntaxKind::BYTE | syntax::SyntaxKind::INT_NUMBER => {
bytes.push(lit.text().to_string())
}
_ => {
return Err(mbe::ExpandError::UnexpectedToken.into());

11
crates/hir-ty/src/tests/simple.rs
View file

@ -2733,14 +2733,3 @@ fn f() {
"#,
);
}
#[test]
fn nested_tuple_index() {
check_no_mismatches(
r#"
fn main() {
let fld: i32 = ((0,),).0.0;
}
"#,
);
}

20
crates/ide-completion/src/completions/dot.rs
View file

@ -793,24 +793,4 @@ fn main() {
",
)
}
#[test]
fn tuple_index_completion() {
check(
r#"
struct I;
impl I {
fn i_method(&self) {}
}
struct S((), I);
fn f(s: S) {
s.1.$0
}
"#,
expect![[r#"
me i_method() fn(&self)
"#]],
);
}
}

4
crates/ide-completion/src/completions/postfix.rs
View file

@ -5,7 +5,7 @@ mod format_like;
use hir::{Documentation, HasAttrs};
use ide_db::{imports::insert_use::ImportScope, ty_filter::TryEnum, SnippetCap};
use syntax::{
ast::{self, AstNode, LiteralKind},
ast::{self, AstNode, AstToken},
SyntaxKind::{EXPR_STMT, STMT_LIST},
TextRange, TextSize,
};
@ -194,7 +194,7 @@ pub(crate) fn complete_postfix(acc: &mut Completions, ctx: &CompletionContext) {
}
if let ast::Expr::Literal(literal) = dot_receiver.clone() {
if let LiteralKind::String(literal_text) = literal.kind() {
if let Some(literal_text) = ast::String::cast(literal.token()) {
add_format_like_completions(acc, ctx, &dot_receiver, cap, &literal_text);
}
}

2
crates/ide-completion/src/context.rs
View file

@ -1062,7 +1062,7 @@ impl<'a> CompletionContext<'a> {
let receiver_is_ambiguous_float_literal = match &receiver {
Some(ast::Expr::Literal(l)) => matches! {
l.kind(),
ast::LiteralKind::FloatNumber { .. } if l.syntax().last_token().map_or(false, |it| it.kind() == T![.])
ast::LiteralKind::FloatNumber { .. } if l.syntax().last_token().map_or(false, |it| it.text().ends_with('.'))
},
_ => false,
};

10
crates/ide/src/syntax_highlighting/highlight.rs
View file

@ -30,15 +30,7 @@ pub(super) fn token(sema: &Semantics<RootDatabase>, token: SyntaxToken) -> Optio
INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => {
SymbolKind::Field.into()
}
INT_NUMBER | FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1
| FLOAT_NUMBER_START_2 => HlTag::NumericLiteral.into(),
DOT if matches!(
token.prev_token().map(|n| n.kind()),
Some(FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2)
) =>
{
HlTag::NumericLiteral.into()
}
INT_NUMBER | FLOAT_NUMBER => HlTag::NumericLiteral.into(),
BYTE => HlTag::ByteLiteral.into(),
CHAR => HlTag::CharLiteral.into(),
IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => {

14
crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
View file

@ -119,13 +119,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">-</span><span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">#</span><span class="numeric_literal">0</span><span class="numeric_literal">10</span><span class="variable">x</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">27</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234</span><span class="numeric_literal">.</span><span class="numeric_literal">56</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234.56</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">&gt;</span><span class="numeric_literal">8</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 right-aligned characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello {{}}"</span><span class="parenthesis">)</span><span class="semicolon">;</span>

79
crates/mbe/src/syntax_bridge.rs
View file

@ -243,8 +243,6 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree {
let char = match token.to_char(conv) {
Some(c) => c,
None => {
// FIXME: this isn't really correct, `to_char` yields the *first* char of the token,
// and this is relevant when eg. creating 2 `tt::Punct` from a single `::` token
panic!("Token from lexer must be single char: token = {:#?}", token);
}
};
@ -262,33 +260,6 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree {
IDENT => make_leaf!(Ident),
UNDERSCORE => make_leaf!(Ident),
k if k.is_keyword() => make_leaf!(Ident),
FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => {
// Reassemble a split-up float token.
let mut range = range;
let mut text = token.to_text(conv).to_string();
if kind == FLOAT_NUMBER_START_1 || kind == FLOAT_NUMBER_START_2 {
let (dot, dot_range) = conv.bump().unwrap();
assert_eq!(dot.kind(conv), DOT);
text += &*dot.to_text(conv);
range = TextRange::new(range.start(), dot_range.end());
if kind == FLOAT_NUMBER_START_2 {
let (tail, tail_range) = conv.bump().unwrap();
assert_eq!(tail.kind(conv), FLOAT_NUMBER_PART);
text += &*tail.to_text(conv);
range = TextRange::new(range.start(), tail_range.end());
}
}
result.push(
tt::Leaf::from(tt::Literal {
id: conv.id_alloc().alloc(range, synth_id),
text: text.into(),
})
.into(),
);
continue;
}
k if k.is_literal() => make_leaf!(Literal),
LIFETIME_IDENT => {
let char_unit = TextSize::of('\'');
@ -742,7 +713,6 @@ struct TtTreeSink<'a> {
text_pos: TextSize,
inner: SyntaxTreeBuilder,
token_map: TokenMap,
remaining_float_lit_text: String,
}
impl<'a> TtTreeSink<'a> {
@ -754,7 +724,6 @@ impl<'a> TtTreeSink<'a> {
text_pos: 0.into(),
inner: SyntaxTreeBuilder::default(),
token_map: TokenMap::default(),
remaining_float_lit_text: String::new(),
}
}
@ -781,54 +750,6 @@ impl<'a> TtTreeSink<'a> {
n_tokens = 2;
}
// We need to split a float `tt::Literal` into up to 3 tokens consumed by the parser.
match self.cursor.token_tree() {
Some(tt::buffer::TokenTreeRef::Subtree(sub, _)) if sub.delimiter.is_none() => {
self.cursor = self.cursor.subtree().unwrap()
}
_ => {}
}
let literal = match self.cursor.token_tree() {
Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Literal(lit), _)) => Some(lit),
_ => None,
};
if matches!(
kind,
FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2
) {
if self.remaining_float_lit_text.is_empty() {
always!(
literal.is_some(),
"kind={:?}, cursor tt={:?}",
kind,
self.cursor.token_tree()
);
let text = literal.map_or(String::new(), |lit| lit.to_string());
self.cursor = self.cursor.bump();
match text.split_once('.') {
Some((start, end)) => {
self.inner.token(kind, start);
self.remaining_float_lit_text = format!(".{end}");
return;
}
None => {
self.inner.token(kind, &text);
return;
}
}
} else {
self.inner.token(kind, &self.remaining_float_lit_text);
self.remaining_float_lit_text.clear();
return;
}
}
if kind == DOT && !self.remaining_float_lit_text.is_empty() {
always!(self.remaining_float_lit_text.chars().next() == Some('.'));
self.inner.token(kind, ".");
self.remaining_float_lit_text = self.remaining_float_lit_text[1..].to_string();
return;
}
let mut last = self.cursor;
for _ in 0..n_tokens {
let tmp: u8;

16
crates/mbe/src/to_parser_input.rs
View file

@ -35,13 +35,15 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input {
let is_negated = lit.text.starts_with('-');
let inner_text = &lit.text[if is_negated { 1 } else { 0 }..];
let lexed_str = parser::LexedStr::new(inner_text);
if lexed_str.is_empty() {
panic!("failed to convert literal: {:?}", lit);
}
for i in 0..lexed_str.len() {
res.push(lexed_str.kind(i));
}
let kind = parser::LexedStr::single_token(inner_text)
.map(|(kind, _error)| kind)
.filter(|kind| {
kind.is_literal()
&& (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER))
})
.unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit));
res.push(kind);
}
tt::Leaf::Ident(ident) => match ident.text.as_ref() {
"_" => res.push(T![_]),

11
crates/mbe/src/tt_iter.rs
View file

@ -90,20 +90,9 @@ impl<'a> TtIter<'a> {
let mut cursor = buffer.begin();
let mut error = false;
let mut float_fragments_to_skip = 0;
for step in tree_traversal.iter() {
match step {
parser::Step::Token { kind, mut n_input_tokens } => {
if float_fragments_to_skip > 0 {
float_fragments_to_skip -= 1;
n_input_tokens = 0;
}
match kind {
SyntaxKind::LIFETIME_IDENT => n_input_tokens = 2,
SyntaxKind::FLOAT_NUMBER_START_1 => float_fragments_to_skip = 1,
SyntaxKind::FLOAT_NUMBER_START_2 => float_fragments_to_skip = 2,
_ => {}
}
if kind == SyntaxKind::LIFETIME_IDENT {
n_input_tokens = 2;
}

13
crates/parser/src/grammar.rs
View file

@ -39,7 +39,6 @@ mod generic_params;
mod types;
use crate::{
grammar::expressions::FLOAT_LITERAL_FIRST,
parser::{CompletedMarker, Marker, Parser},
SyntaxKind::{self, *},
TokenSet, T,
@ -319,17 +318,9 @@ fn name_ref(p: &mut Parser) {
}
fn name_ref_or_index(p: &mut Parser) {
assert!(
p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST)
);
assert!(p.at(IDENT) || p.at(INT_NUMBER));
let m = p.start();
if p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) {
// Ideally we'd remap this to `INT_NUMBER` instead, but that causes the MBE conversion to
// lose track of what's a float and what isn't, causing panics.
p.bump_remap(FLOAT_NUMBER_PART);
} else {
p.bump_any();
}
p.bump_any();
m.complete(p, NAME_REF);
}

12
crates/parser/src/grammar/expressions.rs
View file

@ -3,7 +3,7 @@ mod atom;
use super::*;
pub(crate) use self::atom::{block_expr, match_arm_list};
pub(super) use self::atom::{literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST};
pub(super) use self::atom::{literal, LITERAL_FIRST};
#[derive(PartialEq, Eq)]
pub(super) enum Semicolon {
@ -452,9 +452,6 @@ fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
// fn foo() {
// x.foo();
// y.bar::<T>(1, 2,);
//
// 0e0.sin();
// 0e0f32.sin();
// }
fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
@ -472,16 +469,17 @@ fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
// fn foo() {
// x.foo;
// x.0.bar;
// x.0. bar;
// x.0.1;
// x.0();
// }
fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]));
let m = lhs.precede(p);
p.bump(T![.]);
if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) {
if p.at(IDENT) || p.at(INT_NUMBER) {
name_ref_or_index(p);
} else if p.at(FLOAT_NUMBER) {
// FIXME: How to recover and instead parse INT + T![.]?
p.bump_any();
} else {
p.error("expected field name or number");
}

40
crates/parser/src/grammar/expressions/atom.rs
View file

@ -17,58 +17,22 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[
T![true],
T![false],
INT_NUMBER,
FLOAT_NUMBER_START_0,
FLOAT_NUMBER_START_1,
FLOAT_NUMBER_START_2,
FLOAT_NUMBER,
BYTE,
CHAR,
STRING,
BYTE_STRING,
]);
pub(crate) const FLOAT_LITERAL_FIRST: TokenSet =
TokenSet::new(&[FLOAT_NUMBER_START_0, FLOAT_NUMBER_START_1, FLOAT_NUMBER_START_2]);
pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
if !p.at_ts(LITERAL_FIRST) {
return None;
}
let m = p.start();
if p.at_ts(FLOAT_LITERAL_FIRST) {
float_literal(p);
} else {
// Everything else is just one token.
p.bump_any();
}
p.bump_any();
Some(m.complete(p, LITERAL))
}
// test float_literal
// fn f() {
// 0.0;
// 1.;
// 0e0;
// 0e0f32;
// 1.23f64;
// }
pub(crate) fn float_literal(p: &mut Parser) {
// Floats can be up to 3 tokens. The first token indicates how many there are.
let f = p.start();
if p.at(FLOAT_NUMBER_START_0) {
p.bump(FLOAT_NUMBER_START_0);
} else if p.at(FLOAT_NUMBER_START_1) {
p.bump(FLOAT_NUMBER_START_1);
p.bump(DOT);
} else if p.at(FLOAT_NUMBER_START_2) {
p.bump(FLOAT_NUMBER_START_2);
p.bump(DOT);
p.bump(FLOAT_NUMBER_PART);
} else {
unreachable!();
}
f.complete(p, FLOAT_LITERAL);
}
// E.g. for after the break in `if break {}`, this should not match
pub(super) const ATOM_EXPR_FIRST: TokenSet =
LITERAL_FIRST.union(paths::PATH_FIRST).union(TokenSet::new(&[

2
crates/parser/src/grammar/patterns.rs
View file

@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> {
}
fn is_literal_pat_start(p: &Parser) -> bool {
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART)
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER)
|| p.at_ts(expressions::LITERAL_FIRST)
}

28
crates/parser/src/lexed_str.rs
View file

@ -177,7 +177,7 @@ impl<'a> Converter<'a> {
rustc_lexer::TokenKind::RawIdent => IDENT,
rustc_lexer::TokenKind::Literal { kind, .. } => {
self.extend_literal(token_text, kind);
self.extend_literal(token_text.len(), kind);
return;
}
@ -223,7 +223,7 @@ impl<'a> Converter<'a> {
self.push(syntax_kind, token_text.len(), err);
}
fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) {
fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) {
let mut err = "";
let syntax_kind = match *kind {
@ -237,27 +237,7 @@ impl<'a> Converter<'a> {
if empty_exponent {
err = "Missing digits after the exponent symbol";
}
// In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0`
// is lexed as a float, we split floats that contain a `.` into 3 tokens.
// To ensure that later stages can always reconstruct the token correctly, the first
// token in the sequence indicates the number of following tokens that are part of
// the float literal.
if let Some((before, after)) = token_text.split_once('.') {
let err = if err.is_empty() { None } else { Some(err) };
assert!(!before.is_empty());
let tok =
if after.is_empty() { FLOAT_NUMBER_START_1 } else { FLOAT_NUMBER_START_2 };
self.push(tok, before.len(), None);
self.push(DOT, 1, None);
if !after.is_empty() {
self.push(FLOAT_NUMBER_PART, after.len(), err);
}
return;
}
FLOAT_NUMBER_START_0
FLOAT_NUMBER
}
rustc_lexer::LiteralKind::Char { terminated } => {
if !terminated {
@ -315,6 +295,6 @@ impl<'a> Converter<'a> {
};
let err = if err.is_empty() { None } else { Some(err) };
self.push(syntax_kind, token_text.len(), err);
self.push(syntax_kind, len, err);
}
}

11
crates/parser/src/syntax_kind/generated.rs
View file

File diff suppressed because one or more lines are too long

44
crates/parser/test_data/lexer/err/empty_exponent.rast
View file

@ -1,14 +1,14 @@
FLOAT_NUMBER_START_0 "0e" error: Missing digits after the exponent symbol
FLOAT_NUMBER "0e" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_0 "0E" error: Missing digits after the exponent symbol
FLOAT_NUMBER "0E" error: Missing digits after the exponent symbol
WHITESPACE "\n\n"
FLOAT_NUMBER_START_0 "42e+" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42e+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_0 "42e-" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42e-" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_0 "42E+" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42E+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_0 "42E-" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n"
INT_NUMBER "42"
DOT "."
@ -30,35 +30,19 @@ DOT "."
IDENT "E"
MINUS "-"
WHITESPACE "\n\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2e+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2e-" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2E+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2e+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2e-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2E+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol
FLOAT_NUMBER "42.2E-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"

25
crates/parser/test_data/lexer/ok/numbers.rast
View file

@ -4,8 +4,7 @@ INT_NUMBER "00"
WHITESPACE " "
INT_NUMBER "0_"
WHITESPACE " "
FLOAT_NUMBER_START_1 "0"
DOT "."
FLOAT_NUMBER "0."
WHITESPACE " "
INT_NUMBER "0z"
WHITESPACE "\n"
@ -21,13 +20,11 @@ INT_NUMBER "001279"
WHITESPACE " "
INT_NUMBER "0_1279"
WHITESPACE " "
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "1279"
FLOAT_NUMBER "0.1279"
WHITESPACE " "
FLOAT_NUMBER_START_0 "0e1279"
FLOAT_NUMBER "0e1279"
WHITESPACE " "
FLOAT_NUMBER_START_0 "0E1279"
FLOAT_NUMBER "0E1279"
WHITESPACE "\n"
INT_NUMBER "0"
DOT "."
@ -40,7 +37,7 @@ IDENT "foo"
L_PAREN "("
R_PAREN ")"
WHITESPACE "\n"
FLOAT_NUMBER_START_0 "0e+1"
FLOAT_NUMBER "0e+1"
WHITESPACE "\n"
INT_NUMBER "0"
DOT "."
@ -48,19 +45,13 @@ IDENT "e"
PLUS "+"
INT_NUMBER "1"
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "0E-2"
FLOAT_NUMBER "0.0E-2"
WHITESPACE "\n"
FLOAT_NUMBER_START_2 "0___0"
DOT "."
FLOAT_NUMBER_PART "10000____0000e+111__"
FLOAT_NUMBER "0___0.10000____0000e+111__"
WHITESPACE "\n"
INT_NUMBER "1i64"
WHITESPACE " "
FLOAT_NUMBER_START_2 "92"
DOT "."
FLOAT_NUMBER_PART "0f32"
FLOAT_NUMBER "92.0f32"
WHITESPACE " "
INT_NUMBER "11__s"
WHITESPACE "\n"

4
crates/parser/test_data/parser/err/0023_mismatched_paren.rast
View file

@ -32,9 +32,7 @@ SOURCE_FILE
INT_NUMBER "1"
COMMA ","
WHITESPACE " "
FLOAT_NUMBER_START_2 "2"
DOT "."
FLOAT_NUMBER_PART "0"
FLOAT_NUMBER "2.0"
WHITESPACE "\n "
R_CURLY "}"
WHITESPACE " "

33
crates/parser/test_data/parser/inline/ok/0011_field_expr.rast
View file

@ -40,39 +40,6 @@ SOURCE_FILE
IDENT "bar"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "0"
DOT "."
WHITESPACE " "
NAME_REF
IDENT "bar"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "0"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "1"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
CALL_EXPR
FIELD_EXPR

2
crates/parser/test_data/parser/inline/ok/0011_field_expr.rs
View file

@ -1,7 +1,5 @@
fn foo() {
x.foo;
x.0.bar;
x.0. bar;
x.0.1;
x.0();
}

5
crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast
View file

@ -57,10 +57,7 @@ SOURCE_FILE
EQ "="
WHITESPACE " "
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "2"
DOT "."
FLOAT_NUMBER_PART "0"
FLOAT_NUMBER "2.0"
SEMICOLON ";"
WHITESPACE "\n "
LET_STMT

26
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast
View file

@ -58,32 +58,6 @@ SOURCE_FILE
COMMA ","
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n\n "
EXPR_STMT
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0"
DOT "."
NAME_REF
IDENT "sin"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0f32"
DOT "."
NAME_REF
IDENT "sin"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"

3
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs
View file

@ -1,7 +1,4 @@
fn foo() {
x.foo();
y.bar::<T>(1, 2,);
0e0.sin();
0e0f32.sin();
}

51
crates/parser/test_data/parser/inline/ok/0201_float_literal.rast
View file

@ -1,51 +0,0 @@
SOURCE_FILE
FN
FN_KW "fn"
WHITESPACE " "
NAME
IDENT "f"
PARAM_LIST
L_PAREN "("
R_PAREN ")"
WHITESPACE " "
BLOCK_EXPR
STMT_LIST
L_CURLY "{"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "0"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_1 "1"
DOT "."
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0f32"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "23f64"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"

7
crates/parser/test_data/parser/inline/ok/0201_float_literal.rs
View file

@ -1,7 +0,0 @@
fn f() {
0.0;
1.;
0e0;
0e0f32;
1.23f64;
}

10
crates/parser/test_data/parser/ok/0056_neq_in_type.rast
View file

@ -19,10 +19,7 @@ SOURCE_FILE
CAST_EXPR
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "0f32"
FLOAT_NUMBER "1.0f32"
DOT "."
NAME_REF
IDENT "floor"
@ -43,10 +40,7 @@ SOURCE_FILE
CAST_EXPR
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "0f32"
FLOAT_NUMBER "1.0f32"
DOT "."
NAME_REF
IDENT "floor"

9
crates/syntax/rust.ungram
View file

@ -365,20 +365,13 @@ MacroExpr =
Literal =
Attr* value:(
'int_number' | FloatLiteral
'int_number' | 'float_number'
| 'string' | 'raw_string'
| 'byte_string' | 'raw_byte_string'
| 'true' | 'false'
| 'char' | 'byte'
)
FloatLiteral =
'float_number_start_0'?
'float_number_start_1'?
'float_number_start_2'?
'.'?
'float_number_part'?
PathExpr =
Attr* Path

42
crates/syntax/src/ast/expr_ext.rs
View file

@ -8,7 +8,7 @@ use crate::{
operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp},
support, AstChildren, AstNode,
},
AstToken, SyntaxElement,
AstToken,
SyntaxKind::*,
SyntaxNode, SyntaxToken, T,
};
@ -282,32 +282,30 @@ pub enum LiteralKind {
String(ast::String),
ByteString(ast::ByteString),
IntNumber(ast::IntNumber),
FloatNumber(ast::FloatLiteral),
FloatNumber(ast::FloatNumber),
Char(ast::Char),
Byte(ast::Byte),
Bool(bool),
}
impl ast::Literal {
pub fn value(&self) -> SyntaxElement {
pub fn token(&self) -> SyntaxToken {
self.syntax()
.children_with_tokens()
.find(|e| e.kind() != ATTR && !e.kind().is_trivia())
.and_then(|e| e.into_token())
.unwrap()
}
pub fn kind(&self) -> LiteralKind {
let token = match self.value() {
rowan::NodeOrToken::Node(node) => {
return LiteralKind::FloatNumber(
ast::FloatLiteral::cast(node).expect("unreachable"),
);
}
rowan::NodeOrToken::Token(token) => token,
};
let token = self.token();
if let Some(t) = ast::IntNumber::cast(token.clone()) {
return LiteralKind::IntNumber(t);
}
if let Some(t) = ast::FloatNumber::cast(token.clone()) {
return LiteralKind::FloatNumber(t);
}
if let Some(t) = ast::String::cast(token.clone()) {
return LiteralKind::String(t);
}
@ -327,26 +325,6 @@ impl ast::Literal {
_ => unreachable!(),
}
}
pub fn as_string(&self) -> Option<ast::String> {
match self.kind() {
LiteralKind::String(it) => Some(it),
_ => None,
}
}
pub fn as_byte_string(&self) -> Option<ast::ByteString> {
match self.kind() {
LiteralKind::ByteString(it) => Some(it),
_ => None,
}
}
}
impl ast::FloatLiteral {
pub fn suffix(&self) -> Option<String> {
ast::FloatNumberPart::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string())
}
}
pub enum BlockModifier {
@ -386,7 +364,7 @@ impl ast::BlockExpr {
fn test_literal_with_attr() {
let parse = ast::SourceFile::parse(r#"const _: &str = { #[attr] "Hello" };"#);
let lit = parse.tree().syntax().descendants().find_map(ast::Literal::cast).unwrap();
assert_eq!(lit.value().to_string(), r#""Hello""#);
assert_eq!(lit.token().text(), r#""Hello""#);
}
impl ast::RecordExprField {

36
crates/syntax/src/ast/generated/nodes.rs
View file

@ -1085,26 +1085,6 @@ impl UnderscoreExpr {
pub fn underscore_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![_]) }
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct FloatLiteral {
pub(crate) syntax: SyntaxNode,
}
impl FloatLiteral {
pub fn float_number_start_0_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_0])
}
pub fn float_number_start_1_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_1])
}
pub fn float_number_start_2_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_2])
}
pub fn dot_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![.]) }
pub fn float_number_part_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_part])
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct StmtList {
pub(crate) syntax: SyntaxNode,
@ -2739,17 +2719,6 @@ impl AstNode for UnderscoreExpr {
}
fn syntax(&self) -> &SyntaxNode { &self.syntax }
}
impl AstNode for FloatLiteral {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_LITERAL }
fn cast(syntax: SyntaxNode) -> Option<Self> {
if Self::can_cast(syntax.kind()) {
Some(Self { syntax })
} else {
None
}
}
fn syntax(&self) -> &SyntaxNode { &self.syntax }
}
impl AstNode for StmtList {
fn can_cast(kind: SyntaxKind) -> bool { kind == STMT_LIST }
fn cast(syntax: SyntaxNode) -> Option<Self> {
@ -4639,11 +4608,6 @@ impl std::fmt::Display for UnderscoreExpr {
std::fmt::Display::fmt(self.syntax(), f)
}
}
impl std::fmt::Display for FloatLiteral {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f)
}
}
impl std::fmt::Display for StmtList {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f)

8
crates/syntax/src/ast/generated/tokens.rs
View file

@ -112,16 +112,16 @@ impl AstToken for IntNumber {
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct FloatNumberPart {
pub struct FloatNumber {
pub(crate) syntax: SyntaxToken,
}
impl std::fmt::Display for FloatNumberPart {
impl std::fmt::Display for FloatNumber {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.syntax, f)
}
}
impl AstToken for FloatNumberPart {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER_PART }
impl AstToken for FloatNumber {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER }
fn cast(syntax: SyntaxToken) -> Option<Self> {
if Self::can_cast(syntax.kind()) {
Some(Self { syntax })

13
crates/syntax/src/ast/make.rs
View file

@ -799,11 +799,6 @@ pub fn struct_(
))
}
pub fn literal(text: &str) -> ast::Literal {
assert_eq!(text.trim(), text);
ast_from_text(&format!("fn f() {{ let _ = {}; }}", text))
}
#[track_caller]
fn ast_from_text<N: AstNode>(text: &str) -> N {
let parse = SourceFile::parse(text);
@ -832,7 +827,7 @@ pub fn token(kind: SyntaxKind) -> SyntaxToken {
pub mod tokens {
use once_cell::sync::Lazy;
use crate::{AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken};
use crate::{ast, AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken};
pub(super) static SOURCE_FILE: Lazy<Parse<SourceFile>> = Lazy::new(|| {
SourceFile::parse(
@ -863,6 +858,12 @@ pub mod tokens {
sf.syntax().first_child_or_token().unwrap().into_token().unwrap()
}
pub fn literal(text: &str) -> SyntaxToken {
assert_eq!(text.trim(), text);
let lit: ast::Literal = super::ast_from_text(&format!("fn f() {{ let _ = {}; }}", text));
lit.syntax().first_child_or_token().unwrap().into_token().unwrap()
}
pub fn single_newline() -> SyntaxToken {
let res = SOURCE_FILE
.tree()

4
crates/syntax/src/ast/node_ext.rs
View file

@ -555,9 +555,7 @@ impl ast::FieldExpr {
self.syntax
.children_with_tokens()
// FIXME: Accepting floats here to reject them in validation later
.find(|c| {
c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER_PART
})
.find(|c| c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER)
.as_ref()
.and_then(SyntaxElement::as_token)
.cloned()

29
crates/syntax/src/ast/token_ext.rs
View file

@ -321,7 +321,7 @@ impl ast::IntNumber {
}
}
impl ast::FloatNumberPart {
impl ast::FloatNumber {
pub fn suffix(&self) -> Option<&str> {
let text = self.text();
let mut indices = text.char_indices();
@ -355,24 +355,14 @@ impl Radix {
#[cfg(test)]
mod tests {
use crate::ast::{self, make};
use crate::ast::{self, make, FloatNumber, IntNumber};
fn check_float_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
let suffix = match make::literal(lit).kind() {
ast::LiteralKind::FloatNumber(f) => f.suffix(),
// `1f32` lexes as an INT_NUMBER
ast::LiteralKind::IntNumber(i) => i.suffix().map(|s| s.to_string()),
e => unreachable!("{e:?}"),
};
assert_eq!(suffix.as_deref(), expected.into());
assert_eq!(FloatNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into());
}
fn check_int_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
let i = match make::literal(lit).kind() {
ast::LiteralKind::IntNumber(i) => i,
_ => unreachable!(),
};
assert_eq!(i.suffix(), expected.into());
assert_eq!(IntNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into());
}
#[test]
@ -400,11 +390,12 @@ mod tests {
}
fn check_string_value<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
let s = match make::literal(&format!("\"{}\"", lit)).kind() {
ast::LiteralKind::String(s) => s,
_ => unreachable!(),
};
assert_eq!(s.value().as_deref(), expected.into());
assert_eq!(
ast::String { syntax: make::tokens::literal(&format!("\"{}\"", lit)) }
.value()
.as_deref(),
expected.into()
);
}
#[test]

13
crates/syntax/src/tests/ast_src.rs
View file

@ -71,17 +71,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
"super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield",
],
contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"],
literals: &[
"INT_NUMBER",
"FLOAT_NUMBER_START_0",
"FLOAT_NUMBER_START_1",
"FLOAT_NUMBER_START_2",
"FLOAT_NUMBER_PART",
"CHAR",
"BYTE",
"STRING",
"BYTE_STRING",
],
literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"],
tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"],
nodes: &[
"SOURCE_FILE",
@ -193,7 +183,6 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
"PATH",
"PATH_SEGMENT",
"LITERAL",
"FLOAT_LITERAL",
"RENAME",
"VISIBILITY",
"WHERE_CLAUSE",

6
crates/syntax/src/tests/sourcegen_ast.rs
View file

@ -462,10 +462,6 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String {
[lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT };
[ident] => { $crate::SyntaxKind::IDENT };
[shebang] => { $crate::SyntaxKind::SHEBANG };
[float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART };
[float_number_start_0] => { $crate::SyntaxKind::FLOAT_NUMBER_START_0 };
[float_number_start_1] => { $crate::SyntaxKind::FLOAT_NUMBER_START_1 };
[float_number_start_2] => { $crate::SyntaxKind::FLOAT_NUMBER_START_2 };
}
pub use T;
};
@ -589,7 +585,7 @@ impl Field {
fn lower(grammar: &Grammar) -> AstSrc {
let mut res = AstSrc {
tokens: "Whitespace Comment String ByteString IntNumber FloatNumberPart Char Byte Ident"
tokens: "Whitespace Comment String ByteString IntNumber FloatNumber Char Byte Ident"
.split_ascii_whitespace()
.map(|it| it.to_string())
.collect::<Vec<_>>(),

11
crates/syntax/src/validation.rs
View file

@ -119,15 +119,8 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
text.rfind(end_delimiter).and_then(|end| text.get(prefix_len..end))
}
let token = literal.value();
let text;
let text = match &token {
rowan::NodeOrToken::Node(node) => {
text = node.text().to_string();
&*text
}
rowan::NodeOrToken::Token(token) => token.text(),
};
let token = literal.token();
let text = token.text();
// FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205)
let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| {