Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2025-01-14 14:13:58 +00:00
Code Issues Projects Releases Packages Wiki Activity

Revert "Auto merge of #12149 - jonas-schievink:literally-just-a-literal, r=jonas-schievink"

Browse source 
This reverts commit cc9ae2b89e, reversing
changes made to 7dfd1cb572.
This commit is contained in:
Jonas Schievink 2022-05-13 15:08:14 +02:00
parent 2287ae22c6
commit 9bd11459ba
36 changed files with 121 additions and 502 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
crates/hir-def/src/body/lower.rs
View file

@ -972,7 +972,7 @@ impl From<ast::LiteralKind> for Literal {
} }
} }
LiteralKind::FloatNumber(lit) => { LiteralKind::FloatNumber(lit) => {
let ty = lit.suffix().and_then(|s| BuiltinFloat::from_suffix(&s)); let ty = lit.suffix().and_then(BuiltinFloat::from_suffix);
Literal::Float(Default::default(), ty) Literal::Float(Default::default(), ty)
} }
LiteralKind::ByteString(bs) => { LiteralKind::ByteString(bs) => {

36
crates/hir-expand/src/builtin_fn_macro.rs
View file

@ -4,7 +4,10 @@ use base_db::{AnchoredPath, Edition, FileId};
use cfg::CfgExpr; use cfg::CfgExpr;
use either::Either; use either::Either;
use mbe::{parse_exprs_with_sep, parse_to_token_tree}; use mbe::{parse_exprs_with_sep, parse_to_token_tree};
use syntax::{ast, SmolStr}; use syntax::{
ast::{self, AstToken},
SmolStr,
};
use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc}; use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc};
@ -355,7 +358,14 @@ fn unreachable_expand(
} }
fn unquote_str(lit: &tt::Literal) -> Option<String> { fn unquote_str(lit: &tt::Literal) -> Option<String> {
let token = ast::make::literal(&lit.to_string()).as_string()?; let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::String::cast(lit)?;
token.value().map(|it| it.into_owned())
}
fn unquote_byte_string(lit: &tt::Literal) -> Option<Vec<u8>> {
let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::ByteString::cast(lit)?;
token.value().map(|it| it.into_owned()) token.value().map(|it| it.into_owned())
} }
@ -432,16 +442,12 @@ fn concat_bytes_expand(
for (i, t) in tt.token_trees.iter().enumerate() { for (i, t) in tt.token_trees.iter().enumerate() {
match t { match t {
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
let lit = ast::make::literal(&lit.to_string()); let token = ast::make::tokens::literal(&lit.to_string());
match lit.kind() { match token.kind() {
ast::LiteralKind::ByteString(s) => { syntax::SyntaxKind::BYTE => bytes.push(token.text().to_string()),
s.value() syntax::SyntaxKind::BYTE_STRING => {
.unwrap_or_default() let components = unquote_byte_string(lit).unwrap_or_else(Vec::new);
.into_iter() components.into_iter().for_each(|x| bytes.push(x.to_string()));
.for_each(|x| bytes.push(x.to_string()));
}
ast::LiteralKind::Byte(_) => {
bytes.push(lit.to_string());
} }
_ => { _ => {
err.get_or_insert(mbe::ExpandError::UnexpectedToken.into()); err.get_or_insert(mbe::ExpandError::UnexpectedToken.into());
@ -475,10 +481,10 @@ fn concat_bytes_expand_subtree(
for (ti, tt) in tree.token_trees.iter().enumerate() { for (ti, tt) in tree.token_trees.iter().enumerate() {
match tt { match tt {
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
let lit = ast::make::literal(&lit.to_string()); let lit = ast::make::tokens::literal(&lit.to_string());
match lit.kind() { match lit.kind() {
ast::LiteralKind::IntNumber(_) | ast::LiteralKind::Byte(_) => { syntax::SyntaxKind::BYTE | syntax::SyntaxKind::INT_NUMBER => {
bytes.push(lit.to_string()); bytes.push(lit.text().to_string())
} }
_ => { _ => {
return Err(mbe::ExpandError::UnexpectedToken.into()); return Err(mbe::ExpandError::UnexpectedToken.into());

11
crates/hir-ty/src/tests/simple.rs
View file

@ -2733,14 +2733,3 @@ fn f() {
"#, "#,
); );
} }
#[test]
fn nested_tuple_index() {
check_no_mismatches(
r#"
fn main() {
let fld: i32 = ((0,),).0.0;
}
"#,
);
}

20
crates/ide-completion/src/completions/dot.rs
View file

@ -793,24 +793,4 @@ fn main() {
", ",
) )
} }
#[test]
fn tuple_index_completion() {
check(
r#"
struct I;
impl I {
fn i_method(&self) {}
}
struct S((), I);
fn f(s: S) {
s.1.$0
}
"#,
expect![[r#"
me i_method() fn(&self)
"#]],
);
}
} }

4
crates/ide-completion/src/completions/postfix.rs
View file

@ -5,7 +5,7 @@ mod format_like;
use hir::{Documentation, HasAttrs}; use hir::{Documentation, HasAttrs};
use ide_db::{imports::insert_use::ImportScope, ty_filter::TryEnum, SnippetCap}; use ide_db::{imports::insert_use::ImportScope, ty_filter::TryEnum, SnippetCap};
use syntax::{ use syntax::{
ast::{self, AstNode, LiteralKind}, ast::{self, AstNode, AstToken},
SyntaxKind::{EXPR_STMT, STMT_LIST}, SyntaxKind::{EXPR_STMT, STMT_LIST},
TextRange, TextSize, TextRange, TextSize,
}; };
@ -194,7 +194,7 @@ pub(crate) fn complete_postfix(acc: &mut Completions, ctx: &CompletionContext) {
} }
if let ast::Expr::Literal(literal) = dot_receiver.clone() { if let ast::Expr::Literal(literal) = dot_receiver.clone() {
if let LiteralKind::String(literal_text) = literal.kind() { if let Some(literal_text) = ast::String::cast(literal.token()) {
add_format_like_completions(acc, ctx, &dot_receiver, cap, &literal_text); add_format_like_completions(acc, ctx, &dot_receiver, cap, &literal_text);
} }
} }

10
crates/ide/src/syntax_highlighting/highlight.rs
View file

@ -30,15 +30,7 @@ pub(super) fn token(sema: &Semantics<RootDatabase>, token: SyntaxToken) -> Optio
INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => { INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => {
SymbolKind::Field.into() SymbolKind::Field.into()
} }
INT_NUMBER | FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 INT_NUMBER | FLOAT_NUMBER => HlTag::NumericLiteral.into(),
| FLOAT_NUMBER_START_2 => HlTag::NumericLiteral.into(),
DOT if matches!(
token.prev_token().map(|n| n.kind()),
Some(FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2)
) =>
{
HlTag::NumericLiteral.into()
}
BYTE => HlTag::ByteLiteral.into(), BYTE => HlTag::ByteLiteral.into(),
CHAR => HlTag::CharLiteral.into(), CHAR => HlTag::CharLiteral.into(),
IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => { IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => {

14
crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
View file

@ -119,13 +119,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">-</span><span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">-</span><span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">#</span><span class="numeric_literal">0</span><span class="numeric_literal">10</span><span class="variable">x</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">27</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">#</span><span class="numeric_literal">0</span><span class="numeric_literal">10</span><span class="variable">x</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">27</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234</span><span class="numeric_literal">.</span><span class="numeric_literal">56</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234.56</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">&gt;</span><span class="numeric_literal">8</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 right-aligned characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">&gt;</span><span class="numeric_literal">8</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 right-aligned characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello {{}}"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello {{}}"</span><span class="parenthesis">)</span><span class="semicolon">;</span>

25
crates/mbe/src/syntax_bridge.rs
View file

@ -260,31 +260,6 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree {
IDENT => make_leaf!(Ident), IDENT => make_leaf!(Ident),
UNDERSCORE => make_leaf!(Ident), UNDERSCORE => make_leaf!(Ident),
k if k.is_keyword() => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident),
FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => {
// Reassemble a split-up float token.
let mut range = range;
let mut text = token.to_text(conv).to_string();
if kind == FLOAT_NUMBER_START_1 || kind == FLOAT_NUMBER_START_2 {
let (dot, dot_range) = conv.bump().unwrap();
text += &*dot.to_text(conv);
range = TextRange::new(range.start(), dot_range.end());
if kind == FLOAT_NUMBER_START_2 {
let (tail, tail_range) = conv.bump().unwrap();
text += &*tail.to_text(conv);
range = TextRange::new(range.start(), tail_range.end());
}
}
result.push(
tt::Leaf::from(tt::Literal {
id: conv.id_alloc().alloc(range, synth_id),
text: text.into(),
})
.into(),
);
continue;
}
k if k.is_literal() => make_leaf!(Literal), k if k.is_literal() => make_leaf!(Literal),
LIFETIME_IDENT => { LIFETIME_IDENT => {
let char_unit = TextSize::of('\''); let char_unit = TextSize::of('\'');

16
crates/mbe/src/to_parser_input.rs
View file

@ -35,13 +35,15 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input {
let is_negated = lit.text.starts_with('-'); let is_negated = lit.text.starts_with('-');
let inner_text = &lit.text[if is_negated { 1 } else { 0 }..]; let inner_text = &lit.text[if is_negated { 1 } else { 0 }..];
let lexed_str = parser::LexedStr::new(inner_text); let kind = parser::LexedStr::single_token(inner_text)
if lexed_str.is_empty() { .map(|(kind, _error)| kind)
panic!("failed to convert literal: {:?}", lit); .filter(|kind| {
} kind.is_literal()
for i in 0..lexed_str.len() { && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER))
res.push(lexed_str.kind(i)); })
} .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit));
res.push(kind);
} }
tt::Leaf::Ident(ident) => match ident.text.as_ref() { tt::Leaf::Ident(ident) => match ident.text.as_ref() {
"_" => res.push(T![_]), "_" => res.push(T![_]),

11
crates/parser/src/grammar.rs
View file

@ -39,7 +39,6 @@ mod generic_params;
mod types; mod types;
use crate::{ use crate::{
grammar::expressions::FLOAT_LITERAL_FIRST,
parser::{CompletedMarker, Marker, Parser}, parser::{CompletedMarker, Marker, Parser},
SyntaxKind::{self, *}, SyntaxKind::{self, *},
TokenSet, T, TokenSet, T,
@ -319,15 +318,9 @@ fn name_ref(p: &mut Parser) {
} }
fn name_ref_or_index(p: &mut Parser) { fn name_ref_or_index(p: &mut Parser) {
assert!( assert!(p.at(IDENT) || p.at(INT_NUMBER));
p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST)
);
let m = p.start(); let m = p.start();
if p.at_ts(FLOAT_LITERAL_FIRST) { p.bump_any();
p.bump_remap(FLOAT_NUMBER_PART);
} else {
p.bump_any();
}
m.complete(p, NAME_REF); m.complete(p, NAME_REF);
} }

12
crates/parser/src/grammar/expressions.rs
View file

@ -3,7 +3,7 @@ mod atom;
use super::*; use super::*;
pub(crate) use self::atom::{block_expr, match_arm_list}; pub(crate) use self::atom::{block_expr, match_arm_list};
pub(super) use self::atom::{literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST}; pub(super) use self::atom::{literal, LITERAL_FIRST};
#[derive(PartialEq, Eq)] #[derive(PartialEq, Eq)]
pub(super) enum Semicolon { pub(super) enum Semicolon {
@ -452,9 +452,6 @@ fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
// fn foo() { // fn foo() {
// x.foo(); // x.foo();
// y.bar::<T>(1, 2,); // y.bar::<T>(1, 2,);
//
// 0e0.sin();
// 0e0f32.sin();
// } // }
fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::]))); assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
@ -472,16 +469,17 @@ fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
// fn foo() { // fn foo() {
// x.foo; // x.foo;
// x.0.bar; // x.0.bar;
// x.0. bar;
// x.0.1;
// x.0(); // x.0();
// } // }
fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.])); assert!(p.at(T![.]));
let m = lhs.precede(p); let m = lhs.precede(p);
p.bump(T![.]); p.bump(T![.]);
if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) { if p.at(IDENT) || p.at(INT_NUMBER) {
name_ref_or_index(p); name_ref_or_index(p);
} else if p.at(FLOAT_NUMBER) {
// FIXME: How to recover and instead parse INT + T![.]?
p.bump_any();
} else { } else {
p.error("expected field name or number"); p.error("expected field name or number");
} }

40
crates/parser/src/grammar/expressions/atom.rs
View file

@ -17,58 +17,22 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[
T![true], T![true],
T![false], T![false],
INT_NUMBER, INT_NUMBER,
FLOAT_NUMBER_START_0, FLOAT_NUMBER,
FLOAT_NUMBER_START_1,
FLOAT_NUMBER_START_2,
BYTE, BYTE,
CHAR, CHAR,
STRING, STRING,
BYTE_STRING, BYTE_STRING,
]); ]);
pub(crate) const FLOAT_LITERAL_FIRST: TokenSet =
TokenSet::new(&[FLOAT_NUMBER_START_0, FLOAT_NUMBER_START_1, FLOAT_NUMBER_START_2]);
pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> { pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
if !p.at_ts(LITERAL_FIRST) { if !p.at_ts(LITERAL_FIRST) {
return None; return None;
} }
let m = p.start(); let m = p.start();
if p.at_ts(FLOAT_LITERAL_FIRST) { p.bump_any();
float_literal(p);
} else {
// Everything else is just one token.
p.bump_any();
}
Some(m.complete(p, LITERAL)) Some(m.complete(p, LITERAL))
} }
// test float_literal
// fn f() {
// 0.0;
// 1.;
// 0e0;
// 0e0f32;
// 1.23f64;
// }
pub(crate) fn float_literal(p: &mut Parser) {
// Floats can be up to 3 tokens. The first token indicates how many there are.
let f = p.start();
if p.at(FLOAT_NUMBER_START_0) {
p.bump(FLOAT_NUMBER_START_0);
} else if p.at(FLOAT_NUMBER_START_1) {
p.bump(FLOAT_NUMBER_START_1);
p.bump(DOT);
} else if p.at(FLOAT_NUMBER_START_2) {
p.bump(FLOAT_NUMBER_START_2);
p.bump(DOT);
p.bump(FLOAT_NUMBER_PART);
} else {
unreachable!();
}
f.complete(p, FLOAT_LITERAL);
}
// E.g. for after the break in `if break {}`, this should not match // E.g. for after the break in `if break {}`, this should not match
pub(super) const ATOM_EXPR_FIRST: TokenSet = pub(super) const ATOM_EXPR_FIRST: TokenSet =
LITERAL_FIRST.union(paths::PATH_FIRST).union(TokenSet::new(&[ LITERAL_FIRST.union(paths::PATH_FIRST).union(TokenSet::new(&[

2
crates/parser/src/grammar/patterns.rs
View file

@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> {
} }
fn is_literal_pat_start(p: &Parser) -> bool { fn is_literal_pat_start(p: &Parser) -> bool {
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART) p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER)
|| p.at_ts(expressions::LITERAL_FIRST) || p.at_ts(expressions::LITERAL_FIRST)
} }

28
crates/parser/src/lexed_str.rs
View file

@ -177,7 +177,7 @@ impl<'a> Converter<'a> {
rustc_lexer::TokenKind::RawIdent => IDENT, rustc_lexer::TokenKind::RawIdent => IDENT,
rustc_lexer::TokenKind::Literal { kind, .. } => { rustc_lexer::TokenKind::Literal { kind, .. } => {
self.extend_literal(token_text, kind); self.extend_literal(token_text.len(), kind);
return; return;
} }
@ -223,7 +223,7 @@ impl<'a> Converter<'a> {
self.push(syntax_kind, token_text.len(), err); self.push(syntax_kind, token_text.len(), err);
} }
fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) { fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) {
let mut err = ""; let mut err = "";
let syntax_kind = match *kind { let syntax_kind = match *kind {
@ -237,27 +237,7 @@ impl<'a> Converter<'a> {
if empty_exponent { if empty_exponent {
err = "Missing digits after the exponent symbol"; err = "Missing digits after the exponent symbol";
} }
FLOAT_NUMBER
// In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0`
// is lexed as a float, we split floats that contain a `.` into 3 tokens.
// To ensure that later stages can always reconstruct the token correctly, the first
// token in the sequence indicates the number of following tokens that are part of
// the float literal.
if let Some((before, after)) = token_text.split_once('.') {
let err = if err.is_empty() { None } else { Some(err) };
assert!(!before.is_empty());
let tok =
if after.is_empty() { FLOAT_NUMBER_START_1 } else { FLOAT_NUMBER_START_2 };
self.push(tok, before.len(), None);
self.push(DOT, 1, None);
if !after.is_empty() {
self.push(FLOAT_NUMBER_PART, after.len(), err);
}
return;
}
FLOAT_NUMBER_START_0
} }
rustc_lexer::LiteralKind::Char { terminated } => { rustc_lexer::LiteralKind::Char { terminated } => {
if !terminated { if !terminated {
@ -315,6 +295,6 @@ impl<'a> Converter<'a> {
}; };
let err = if err.is_empty() { None } else { Some(err) }; let err = if err.is_empty() { None } else { Some(err) };
self.push(syntax_kind, token_text.len(), err); self.push(syntax_kind, len, err);
} }
} }

11
crates/parser/src/syntax_kind/generated.rs
View file

File diff suppressed because one or more lines are too long

44
crates/parser/test_data/lexer/err/empty_exponent.rast
View file

@ -1,14 +1,14 @@
FLOAT_NUMBER_START_0 "0e" error: Missing digits after the exponent symbol FLOAT_NUMBER "0e" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_0 "0E" error: Missing digits after the exponent symbol FLOAT_NUMBER "0E" error: Missing digits after the exponent symbol
WHITESPACE "\n\n" WHITESPACE "\n\n"
FLOAT_NUMBER_START_0 "42e+" error: Missing digits after the exponent symbol FLOAT_NUMBER "42e+" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_0 "42e-" error: Missing digits after the exponent symbol FLOAT_NUMBER "42e-" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_0 "42E+" error: Missing digits after the exponent symbol FLOAT_NUMBER "42E+" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_0 "42E-" error: Missing digits after the exponent symbol FLOAT_NUMBER "42E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n" WHITESPACE "\n\n"
INT_NUMBER "42" INT_NUMBER "42"
DOT "." DOT "."
@ -30,35 +30,19 @@ DOT "."
IDENT "E" IDENT "E"
MINUS "-" MINUS "-"
WHITESPACE "\n\n" WHITESPACE "\n\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2e+" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2e-" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2E+" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2E-" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n" WHITESPACE "\n\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2e+f32" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2e-f32" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2E+f32" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "42" FLOAT_NUMBER "42.2E-f32" error: Missing digits after the exponent symbol
DOT "."
FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"

25
crates/parser/test_data/lexer/ok/numbers.rast
View file

@ -4,8 +4,7 @@ INT_NUMBER "00"
WHITESPACE " " WHITESPACE " "
INT_NUMBER "0_" INT_NUMBER "0_"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER_START_1 "0" FLOAT_NUMBER "0."
DOT "."
WHITESPACE " " WHITESPACE " "
INT_NUMBER "0z" INT_NUMBER "0z"
WHITESPACE "\n" WHITESPACE "\n"
@ -21,13 +20,11 @@ INT_NUMBER "001279"
WHITESPACE " " WHITESPACE " "
INT_NUMBER "0_1279" INT_NUMBER "0_1279"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER_START_2 "0" FLOAT_NUMBER "0.1279"
DOT "."
FLOAT_NUMBER_PART "1279"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER_START_0 "0e1279" FLOAT_NUMBER "0e1279"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER_START_0 "0E1279" FLOAT_NUMBER "0E1279"
WHITESPACE "\n" WHITESPACE "\n"
INT_NUMBER "0" INT_NUMBER "0"
DOT "." DOT "."
@ -40,7 +37,7 @@ IDENT "foo"
L_PAREN "(" L_PAREN "("
R_PAREN ")" R_PAREN ")"
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_0 "0e+1" FLOAT_NUMBER "0e+1"
WHITESPACE "\n" WHITESPACE "\n"
INT_NUMBER "0" INT_NUMBER "0"
DOT "." DOT "."
@ -48,19 +45,13 @@ IDENT "e"
PLUS "+" PLUS "+"
INT_NUMBER "1" INT_NUMBER "1"
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "0" FLOAT_NUMBER "0.0E-2"
DOT "."
FLOAT_NUMBER_PART "0E-2"
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER_START_2 "0___0" FLOAT_NUMBER "0___0.10000____0000e+111__"
DOT "."
FLOAT_NUMBER_PART "10000____0000e+111__"
WHITESPACE "\n" WHITESPACE "\n"
INT_NUMBER "1i64" INT_NUMBER "1i64"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER_START_2 "92" FLOAT_NUMBER "92.0f32"
DOT "."
FLOAT_NUMBER_PART "0f32"
WHITESPACE " " WHITESPACE " "
INT_NUMBER "11__s" INT_NUMBER "11__s"
WHITESPACE "\n" WHITESPACE "\n"

4
crates/parser/test_data/parser/err/0023_mismatched_paren.rast
View file

@ -32,9 +32,7 @@ SOURCE_FILE
INT_NUMBER "1" INT_NUMBER "1"
COMMA "," COMMA ","
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER_START_2 "2" FLOAT_NUMBER "2.0"
DOT "."
FLOAT_NUMBER_PART "0"
WHITESPACE "\n " WHITESPACE "\n "
R_CURLY "}" R_CURLY "}"
WHITESPACE " " WHITESPACE " "

33
crates/parser/test_data/parser/inline/ok/0011_field_expr.rast
View file

@ -40,39 +40,6 @@ SOURCE_FILE
IDENT "bar" IDENT "bar"
SEMICOLON ";" SEMICOLON ";"
WHITESPACE "\n " WHITESPACE "\n "
EXPR_STMT
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "0"
DOT "."
WHITESPACE " "
NAME_REF
IDENT "bar"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "0"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "1"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT EXPR_STMT
CALL_EXPR CALL_EXPR
FIELD_EXPR FIELD_EXPR

2
crates/parser/test_data/parser/inline/ok/0011_field_expr.rs
View file

@ -1,7 +1,5 @@
fn foo() { fn foo() {
x.foo; x.foo;
x.0.bar; x.0.bar;
x.0. bar;
x.0.1;
x.0(); x.0();
} }

5
crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast
View file

@ -57,10 +57,7 @@ SOURCE_FILE
EQ "=" EQ "="
WHITESPACE " " WHITESPACE " "
LITERAL LITERAL
FLOAT_LITERAL FLOAT_NUMBER "2.0"
FLOAT_NUMBER_START_2 "2"
DOT "."
FLOAT_NUMBER_PART "0"
SEMICOLON ";" SEMICOLON ";"
WHITESPACE "\n " WHITESPACE "\n "
LET_STMT LET_STMT

26
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast
View file

@ -58,32 +58,6 @@ SOURCE_FILE
COMMA "," COMMA ","
R_PAREN ")" R_PAREN ")"
SEMICOLON ";" SEMICOLON ";"
WHITESPACE "\n\n "
EXPR_STMT
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0"
DOT "."
NAME_REF
IDENT "sin"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0f32"
DOT "."
NAME_REF
IDENT "sin"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n" WHITESPACE "\n"
R_CURLY "}" R_CURLY "}"
WHITESPACE "\n" WHITESPACE "\n"

3
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs
View file

@ -1,7 +1,4 @@
fn foo() { fn foo() {
x.foo(); x.foo();
y.bar::<T>(1, 2,); y.bar::<T>(1, 2,);
0e0.sin();
0e0f32.sin();
} }

51
crates/parser/test_data/parser/inline/ok/0201_float_literal.rast
View file

@ -1,51 +0,0 @@
SOURCE_FILE
FN
FN_KW "fn"
WHITESPACE " "
NAME
IDENT "f"
PARAM_LIST
L_PAREN "("
R_PAREN ")"
WHITESPACE " "
BLOCK_EXPR
STMT_LIST
L_CURLY "{"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "0"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_1 "1"
DOT "."
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0f32"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "23f64"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"

7
crates/parser/test_data/parser/inline/ok/0201_float_literal.rs
View file

@ -1,7 +0,0 @@
fn f() {
0.0;
1.;
0e0;
0e0f32;
1.23f64;
}

10
crates/parser/test_data/parser/ok/0056_neq_in_type.rast
View file

@ -19,10 +19,7 @@ SOURCE_FILE
CAST_EXPR CAST_EXPR
METHOD_CALL_EXPR METHOD_CALL_EXPR
LITERAL LITERAL
FLOAT_LITERAL FLOAT_NUMBER "1.0f32"
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "0f32"
DOT "." DOT "."
NAME_REF NAME_REF
IDENT "floor" IDENT "floor"
@ -43,10 +40,7 @@ SOURCE_FILE
CAST_EXPR CAST_EXPR
METHOD_CALL_EXPR METHOD_CALL_EXPR
LITERAL LITERAL
FLOAT_LITERAL FLOAT_NUMBER "1.0f32"
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "0f32"
DOT "." DOT "."
NAME_REF NAME_REF
IDENT "floor" IDENT "floor"

9
crates/syntax/rust.ungram
View file

@ -365,20 +365,13 @@ MacroExpr =
Literal = Literal =
Attr* value:( Attr* value:(
'int_number' | FloatLiteral 'int_number' | 'float_number'
| 'string' | 'raw_string' | 'string' | 'raw_string'
| 'byte_string' | 'raw_byte_string' | 'byte_string' | 'raw_byte_string'
| 'true' | 'false' | 'true' | 'false'
| 'char' | 'byte' | 'char' | 'byte'
) )
FloatLiteral =
'float_number_start_0'?
'float_number_start_1'?
'float_number_start_2'?
'.'?
'float_number_part'?
PathExpr = PathExpr =
Attr* Path Attr* Path

42
crates/syntax/src/ast/expr_ext.rs
View file

@ -8,7 +8,7 @@ use crate::{
operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp}, operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp},
support, AstChildren, AstNode, support, AstChildren, AstNode,
}, },
AstToken, SyntaxElement, AstToken,
SyntaxKind::*, SyntaxKind::*,
SyntaxNode, SyntaxToken, T, SyntaxNode, SyntaxToken, T,
}; };
@ -282,32 +282,30 @@ pub enum LiteralKind {
String(ast::String), String(ast::String),
ByteString(ast::ByteString), ByteString(ast::ByteString),
IntNumber(ast::IntNumber), IntNumber(ast::IntNumber),
FloatNumber(ast::FloatLiteral), FloatNumber(ast::FloatNumber),
Char(ast::Char), Char(ast::Char),
Byte(ast::Byte), Byte(ast::Byte),
Bool(bool), Bool(bool),
} }
impl ast::Literal { impl ast::Literal {
pub fn value(&self) -> SyntaxElement { pub fn token(&self) -> SyntaxToken {
self.syntax() self.syntax()
.children_with_tokens() .children_with_tokens()
.find(|e| e.kind() != ATTR && !e.kind().is_trivia()) .find(|e| e.kind() != ATTR && !e.kind().is_trivia())
.and_then(|e| e.into_token())
.unwrap() .unwrap()
} }
pub fn kind(&self) -> LiteralKind { pub fn kind(&self) -> LiteralKind {
let token = match self.value() { let token = self.token();
rowan::NodeOrToken::Node(node) => {
return LiteralKind::FloatNumber(
ast::FloatLiteral::cast(node).expect("unreachable"),
);
}
rowan::NodeOrToken::Token(token) => token,
};
if let Some(t) = ast::IntNumber::cast(token.clone()) { if let Some(t) = ast::IntNumber::cast(token.clone()) {
return LiteralKind::IntNumber(t); return LiteralKind::IntNumber(t);
} }
if let Some(t) = ast::FloatNumber::cast(token.clone()) {
return LiteralKind::FloatNumber(t);
}
if let Some(t) = ast::String::cast(token.clone()) { if let Some(t) = ast::String::cast(token.clone()) {
return LiteralKind::String(t); return LiteralKind::String(t);
} }
@ -327,26 +325,6 @@ impl ast::Literal {
_ => unreachable!(), _ => unreachable!(),
} }
} }
pub fn as_string(&self) -> Option<ast::String> {
match self.kind() {
LiteralKind::String(it) => Some(it),
_ => None,
}
}
pub fn as_byte_string(&self) -> Option<ast::ByteString> {
match self.kind() {
LiteralKind::ByteString(it) => Some(it),
_ => None,
}
}
}
impl ast::FloatLiteral {
pub fn suffix(&self) -> Option<String> {
ast::FloatNumberPart::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string())
}
} }
pub enum BlockModifier { pub enum BlockModifier {
@ -386,7 +364,7 @@ impl ast::BlockExpr {
fn test_literal_with_attr() { fn test_literal_with_attr() {
let parse = ast::SourceFile::parse(r#"const _: &str = { #[attr] "Hello" };"#); let parse = ast::SourceFile::parse(r#"const _: &str = { #[attr] "Hello" };"#);
let lit = parse.tree().syntax().descendants().find_map(ast::Literal::cast).unwrap(); let lit = parse.tree().syntax().descendants().find_map(ast::Literal::cast).unwrap();
assert_eq!(lit.value().to_string(), r#""Hello""#); assert_eq!(lit.token().text(), r#""Hello""#);
} }
impl ast::RecordExprField { impl ast::RecordExprField {

36
crates/syntax/src/ast/generated/nodes.rs
View file

@ -1085,26 +1085,6 @@ impl UnderscoreExpr {
pub fn underscore_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![_]) } pub fn underscore_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![_]) }
} }
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct FloatLiteral {
pub(crate) syntax: SyntaxNode,
}
impl FloatLiteral {
pub fn float_number_start_0_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_0])
}
pub fn float_number_start_1_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_1])
}
pub fn float_number_start_2_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_2])
}
pub fn dot_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![.]) }
pub fn float_number_part_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_part])
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct StmtList { pub struct StmtList {
pub(crate) syntax: SyntaxNode, pub(crate) syntax: SyntaxNode,
@ -2739,17 +2719,6 @@ impl AstNode for UnderscoreExpr {
} }
fn syntax(&self) -> &SyntaxNode { &self.syntax } fn syntax(&self) -> &SyntaxNode { &self.syntax }
} }
impl AstNode for FloatLiteral {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_LITERAL }
fn cast(syntax: SyntaxNode) -> Option<Self> {
if Self::can_cast(syntax.kind()) {
Some(Self { syntax })
} else {
None
}
}
fn syntax(&self) -> &SyntaxNode { &self.syntax }
}
impl AstNode for StmtList { impl AstNode for StmtList {
fn can_cast(kind: SyntaxKind) -> bool { kind == STMT_LIST } fn can_cast(kind: SyntaxKind) -> bool { kind == STMT_LIST }
fn cast(syntax: SyntaxNode) -> Option<Self> { fn cast(syntax: SyntaxNode) -> Option<Self> {
@ -4639,11 +4608,6 @@ impl std::fmt::Display for UnderscoreExpr {
std::fmt::Display::fmt(self.syntax(), f) std::fmt::Display::fmt(self.syntax(), f)
} }
} }
impl std::fmt::Display for FloatLiteral {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f)
}
}
impl std::fmt::Display for StmtList { impl std::fmt::Display for StmtList {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f) std::fmt::Display::fmt(self.syntax(), f)

8
crates/syntax/src/ast/generated/tokens.rs
View file

@ -112,16 +112,16 @@ impl AstToken for IntNumber {
} }
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct FloatNumberPart { pub struct FloatNumber {
pub(crate) syntax: SyntaxToken, pub(crate) syntax: SyntaxToken,
} }
impl std::fmt::Display for FloatNumberPart { impl std::fmt::Display for FloatNumber {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.syntax, f) std::fmt::Display::fmt(&self.syntax, f)
} }
} }
impl AstToken for FloatNumberPart { impl AstToken for FloatNumber {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER_PART } fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER }
fn cast(syntax: SyntaxToken) -> Option<Self> { fn cast(syntax: SyntaxToken) -> Option<Self> {
if Self::can_cast(syntax.kind()) { if Self::can_cast(syntax.kind()) {
Some(Self { syntax }) Some(Self { syntax })

13
crates/syntax/src/ast/make.rs
View file

@ -799,11 +799,6 @@ pub fn struct_(
)) ))
} }
pub fn literal(text: &str) -> ast::Literal {
assert_eq!(text.trim(), text);
ast_from_text(&format!("fn f() {{ let _ = {}; }}", text))
}
#[track_caller] #[track_caller]
fn ast_from_text<N: AstNode>(text: &str) -> N { fn ast_from_text<N: AstNode>(text: &str) -> N {
let parse = SourceFile::parse(text); let parse = SourceFile::parse(text);
@ -832,7 +827,7 @@ pub fn token(kind: SyntaxKind) -> SyntaxToken {
pub mod tokens { pub mod tokens {
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use crate::{AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken}; use crate::{ast, AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken};
pub(super) static SOURCE_FILE: Lazy<Parse<SourceFile>> = Lazy::new(|| { pub(super) static SOURCE_FILE: Lazy<Parse<SourceFile>> = Lazy::new(|| {
SourceFile::parse( SourceFile::parse(
@ -863,6 +858,12 @@ pub mod tokens {
sf.syntax().first_child_or_token().unwrap().into_token().unwrap() sf.syntax().first_child_or_token().unwrap().into_token().unwrap()
} }
pub fn literal(text: &str) -> SyntaxToken {
assert_eq!(text.trim(), text);
let lit: ast::Literal = super::ast_from_text(&format!("fn f() {{ let _ = {}; }}", text));
lit.syntax().first_child_or_token().unwrap().into_token().unwrap()
}
pub fn single_newline() -> SyntaxToken { pub fn single_newline() -> SyntaxToken {
let res = SOURCE_FILE let res = SOURCE_FILE
.tree() .tree()

4
crates/syntax/src/ast/node_ext.rs
View file

@ -555,9 +555,7 @@ impl ast::FieldExpr {
self.syntax self.syntax
.children_with_tokens() .children_with_tokens()
// FIXME: Accepting floats here to reject them in validation later // FIXME: Accepting floats here to reject them in validation later
.find(|c| { .find(|c| c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER)
c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER_PART
})
.as_ref() .as_ref()
.and_then(SyntaxElement::as_token) .and_then(SyntaxElement::as_token)
.cloned() .cloned()

29
crates/syntax/src/ast/token_ext.rs
View file

@ -321,7 +321,7 @@ impl ast::IntNumber {
} }
} }
impl ast::FloatNumberPart { impl ast::FloatNumber {
pub fn suffix(&self) -> Option<&str> { pub fn suffix(&self) -> Option<&str> {
let text = self.text(); let text = self.text();
let mut indices = text.char_indices(); let mut indices = text.char_indices();
@ -355,24 +355,14 @@ impl Radix {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::ast::{self, make}; use crate::ast::{self, make, FloatNumber, IntNumber};
fn check_float_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) { fn check_float_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
let suffix = match make::literal(lit).kind() { assert_eq!(FloatNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into());
ast::LiteralKind::FloatNumber(f) => f.suffix(),
// `1f32` lexes as an INT_NUMBER
ast::LiteralKind::IntNumber(i) => i.suffix().map(|s| s.to_string()),
e => unreachable!("{e:?}"),
};
assert_eq!(suffix.as_deref(), expected.into());
} }
fn check_int_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) { fn check_int_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
let i = match make::literal(lit).kind() { assert_eq!(IntNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into());
ast::LiteralKind::IntNumber(i) => i,
_ => unreachable!(),
};
assert_eq!(i.suffix(), expected.into());
} }
#[test] #[test]
@ -400,11 +390,12 @@ mod tests {
} }
fn check_string_value<'a>(lit: &str, expected: impl Into<Option<&'a str>>) { fn check_string_value<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
let s = match make::literal(&format!("\"{}\"", lit)).kind() { assert_eq!(
ast::LiteralKind::String(s) => s, ast::String { syntax: make::tokens::literal(&format!("\"{}\"", lit)) }
_ => unreachable!(), .value()
}; .as_deref(),
assert_eq!(s.value().as_deref(), expected.into()); expected.into()
);
} }
#[test] #[test]

13
crates/syntax/src/tests/ast_src.rs
View file

@ -71,17 +71,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
"super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield", "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield",
], ],
contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"], contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"],
literals: &[ literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"],
"INT_NUMBER",
"FLOAT_NUMBER_START_0",
"FLOAT_NUMBER_START_1",
"FLOAT_NUMBER_START_2",
"FLOAT_NUMBER_PART",
"CHAR",
"BYTE",
"STRING",
"BYTE_STRING",
],
tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"], tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"],
nodes: &[ nodes: &[
"SOURCE_FILE", "SOURCE_FILE",
@ -193,7 +183,6 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
"PATH", "PATH",
"PATH_SEGMENT", "PATH_SEGMENT",
"LITERAL", "LITERAL",
"FLOAT_LITERAL",
"RENAME", "RENAME",
"VISIBILITY", "VISIBILITY",
"WHERE_CLAUSE", "WHERE_CLAUSE",

6
crates/syntax/src/tests/sourcegen_ast.rs
View file

@ -462,10 +462,6 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String {
[lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT }; [lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT };
[ident] => { $crate::SyntaxKind::IDENT }; [ident] => { $crate::SyntaxKind::IDENT };
[shebang] => { $crate::SyntaxKind::SHEBANG }; [shebang] => { $crate::SyntaxKind::SHEBANG };
[float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART };
[float_number_start_0] => { $crate::SyntaxKind::FLOAT_NUMBER_START_0 };
[float_number_start_1] => { $crate::SyntaxKind::FLOAT_NUMBER_START_1 };
[float_number_start_2] => { $crate::SyntaxKind::FLOAT_NUMBER_START_2 };
} }
pub use T; pub use T;
}; };
@ -589,7 +585,7 @@ impl Field {
fn lower(grammar: &Grammar) -> AstSrc { fn lower(grammar: &Grammar) -> AstSrc {
let mut res = AstSrc { let mut res = AstSrc {
tokens: "Whitespace Comment String ByteString IntNumber FloatNumberPart Char Byte Ident" tokens: "Whitespace Comment String ByteString IntNumber FloatNumber Char Byte Ident"
.split_ascii_whitespace() .split_ascii_whitespace()
.map(|it| it.to_string()) .map(|it| it.to_string())
.collect::<Vec<_>>(), .collect::<Vec<_>>(),

11
crates/syntax/src/validation.rs
View file

@ -119,15 +119,8 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
text.rfind(end_delimiter).and_then(|end| text.get(prefix_len..end)) text.rfind(end_delimiter).and_then(|end| text.get(prefix_len..end))
} }
let token = literal.value(); let token = literal.token();
let text; let text = token.text();
let text = match &token {
rowan::NodeOrToken::Node(node) => {
text = node.text().to_string();
&*text
}
rowan::NodeOrToken::Token(token) => token.text(),
};
// FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205) // FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205)
let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| { let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| {