Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-12-26 13:03:31 +00:00
Code Issues Projects Releases Packages Wiki Activity

Auto merge of #12149 - jonas-schievink:literally-just-a-literal, r=jonas-schievink

Browse source 
fix: split float literal tokens at `.` to fix parsing of tuple field accesses

This introduces an `ast::FloatLiteral` node, changes the `FLOAT_LITERAL` token to `FLOAT_LITERAL_PART`, and splits any float literal at the `.` character, into a `FLOAT_LITERAL_PART`, and optional `DOT` and trailing `FLOAT_LITERAL_PART` token. The tokens are reassembled when passing them to a macro as a `tt::Literal`.

~~A slight regression is introduced in how float literals are highlighted: the `.` is now highlighted as an operator. I've tried to fix this but couldn't figure out how to highlight the whole `ast::FloatLiteral` node as a unit.~~ This is fixed

Fixes https://github.com/rust-lang/rust-analyzer/issues/1109
Fixes https://github.com/rust-lang/rust-analyzer/issues/10492
Fixes https://github.com/rust-lang/rust-analyzer/issues/12107
Fixes https://github.com/rust-lang/rust-analyzer/issues/10560
Fixes https://github.com/rust-lang/rust-analyzer/issues/11487
This commit is contained in:
bors 2022-05-05 15:35:10 +00:00
commit cc9ae2b89e
37 changed files with 503 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
crates/hir-def/src/body/lower.rs
View file

@ -972,7 +972,7 @@ impl From<ast::LiteralKind> for Literal {
}
}
LiteralKind::FloatNumber(lit) => {
let ty = lit.suffix().and_then(BuiltinFloat::from_suffix);
let ty = lit.suffix().and_then(|s| BuiltinFloat::from_suffix(&s));
Literal::Float(Default::default(), ty)
}
LiteralKind::ByteString(bs) => {

36
crates/hir-expand/src/builtin_fn_macro.rs
View file

@ -4,10 +4,7 @@ use base_db::{AnchoredPath, Edition, FileId};
use cfg::CfgExpr;
use either::Either;
use mbe::{parse_exprs_with_sep, parse_to_token_tree};
use syntax::{
ast::{self, AstToken},
SmolStr,
};
use syntax::{ast, SmolStr};
use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc};
@ -358,14 +355,7 @@ fn unreachable_expand(
}
fn unquote_str(lit: &tt::Literal) -> Option<String> {
let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::String::cast(lit)?;
token.value().map(|it| it.into_owned())
}
fn unquote_byte_string(lit: &tt::Literal) -> Option<Vec<u8>> {
let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::ByteString::cast(lit)?;
let token = ast::make::literal(&lit.to_string()).as_string()?;
token.value().map(|it| it.into_owned())
}
@ -442,12 +432,16 @@ fn concat_bytes_expand(
for (i, t) in tt.token_trees.iter().enumerate() {
match t {
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
let token = ast::make::tokens::literal(&lit.to_string());
match token.kind() {
syntax::SyntaxKind::BYTE => bytes.push(token.text().to_string()),
syntax::SyntaxKind::BYTE_STRING => {
let components = unquote_byte_string(lit).unwrap_or_else(Vec::new);
components.into_iter().for_each(|x| bytes.push(x.to_string()));
let lit = ast::make::literal(&lit.to_string());
match lit.kind() {
ast::LiteralKind::ByteString(s) => {
s.value()
.unwrap_or_default()
.into_iter()
.for_each(|x| bytes.push(x.to_string()));
}
ast::LiteralKind::Byte(_) => {
bytes.push(lit.to_string());
}
_ => {
err.get_or_insert(mbe::ExpandError::UnexpectedToken.into());
@ -481,10 +475,10 @@ fn concat_bytes_expand_subtree(
for (ti, tt) in tree.token_trees.iter().enumerate() {
match tt {
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
let lit = ast::make::tokens::literal(&lit.to_string());
let lit = ast::make::literal(&lit.to_string());
match lit.kind() {
syntax::SyntaxKind::BYTE | syntax::SyntaxKind::INT_NUMBER => {
bytes.push(lit.text().to_string())
ast::LiteralKind::IntNumber(_) | ast::LiteralKind::Byte(_) => {
bytes.push(lit.to_string());
}
_ => {
return Err(mbe::ExpandError::UnexpectedToken.into());

11
crates/hir-ty/src/tests/simple.rs
View file

@ -2733,3 +2733,14 @@ fn f() {
"#,
);
}
#[test]
fn nested_tuple_index() {
check_no_mismatches(
r#"
fn main() {
let fld: i32 = ((0,),).0.0;
}
"#,
);
}

20
crates/ide-completion/src/completions/dot.rs
View file

@ -785,4 +785,24 @@ fn main() {
",
)
}
#[test]
fn tuple_index_completion() {
check(
r#"
struct I;
impl I {
fn i_method(&self) {}
}
struct S((), I);
fn f(s: S) {
s.1.$0
}
"#,
expect![[r#"
me i_method() fn(&self)
"#]],
);
}
}

4
crates/ide-completion/src/completions/postfix.rs
View file

@ -5,7 +5,7 @@ mod format_like;
use hir::{Documentation, HasAttrs};
use ide_db::{imports::insert_use::ImportScope, ty_filter::TryEnum, SnippetCap};
use syntax::{
ast::{self, AstNode, AstToken},
ast::{self, AstNode, LiteralKind},
SyntaxKind::{EXPR_STMT, STMT_LIST},
TextRange, TextSize,
};
@ -191,7 +191,7 @@ pub(crate) fn complete_postfix(acc: &mut Completions, ctx: &CompletionContext) {
}
if let ast::Expr::Literal(literal) = dot_receiver.clone() {
if let Some(literal_text) = ast::String::cast(literal.token()) {
if let LiteralKind::String(literal_text) = literal.kind() {
add_format_like_completions(acc, ctx, &dot_receiver, cap, &literal_text);
}
}

2
crates/ide-completion/src/patterns.rs
View file

@ -230,7 +230,7 @@ pub(crate) fn determine_location(
let receiver = find_in_original_file(it.expr(), original_file);
let receiver_is_ambiguous_float_literal = if let Some(ast::Expr::Literal(l)) = &receiver {
match l.kind() {
ast::LiteralKind::FloatNumber { .. } => l.token().text().ends_with('.'),
ast::LiteralKind::FloatNumber { .. } => l.to_string().ends_with('.'),
_ => false,
}
} else {

10
crates/ide/src/syntax_highlighting/highlight.rs
View file

@ -30,7 +30,15 @@ pub(super) fn token(sema: &Semantics<RootDatabase>, token: SyntaxToken) -> Optio
INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => {
SymbolKind::Field.into()
}
INT_NUMBER | FLOAT_NUMBER => HlTag::NumericLiteral.into(),
INT_NUMBER | FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1
| FLOAT_NUMBER_START_2 => HlTag::NumericLiteral.into(),
DOT if matches!(
token.prev_token().map(|n| n.kind()),
Some(FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2)
) =>
{
HlTag::NumericLiteral.into()
}
BYTE => HlTag::ByteLiteral.into(),
CHAR => HlTag::CharLiteral.into(),
IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => {

14
crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
View file

@ -119,13 +119,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">-</span><span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">#</span><span class="numeric_literal">0</span><span class="numeric_literal">10</span><span class="variable">x</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">27</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234.56</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234</span><span class="numeric_literal">.</span><span class="numeric_literal">56</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">&gt;</span><span class="numeric_literal">8</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 right-aligned characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello {{}}"</span><span class="parenthesis">)</span><span class="semicolon">;</span>

25
crates/mbe/src/syntax_bridge.rs
View file

@ -260,6 +260,31 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree {
IDENT => make_leaf!(Ident),
UNDERSCORE => make_leaf!(Ident),
k if k.is_keyword() => make_leaf!(Ident),
FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => {
// Reassemble a split-up float token.
let mut range = range;
let mut text = token.to_text(conv).to_string();
if kind == FLOAT_NUMBER_START_1 || kind == FLOAT_NUMBER_START_2 {
let (dot, dot_range) = conv.bump().unwrap();
text += &*dot.to_text(conv);
range = TextRange::new(range.start(), dot_range.end());
if kind == FLOAT_NUMBER_START_2 {
let (tail, tail_range) = conv.bump().unwrap();
text += &*tail.to_text(conv);
range = TextRange::new(range.start(), tail_range.end());
}
}
result.push(
tt::Leaf::from(tt::Literal {
id: conv.id_alloc().alloc(range, synth_id),
text: text.into(),
})
.into(),
);
continue;
}
k if k.is_literal() => make_leaf!(Literal),
LIFETIME_IDENT => {
let char_unit = TextSize::of('\'');

16
crates/mbe/src/to_parser_input.rs
View file

@ -35,15 +35,13 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input {
let is_negated = lit.text.starts_with('-');
let inner_text = &lit.text[if is_negated { 1 } else { 0 }..];
let kind = parser::LexedStr::single_token(inner_text)
.map(|(kind, _error)| kind)
.filter(|kind| {
kind.is_literal()
&& (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER))
})
.unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit));
res.push(kind);
let lexed_str = parser::LexedStr::new(inner_text);
if lexed_str.is_empty() {
panic!("failed to convert literal: {:?}", lit);
}
for i in 0..lexed_str.len() {
res.push(lexed_str.kind(i));
}
}
tt::Leaf::Ident(ident) => match ident.text.as_ref() {
"_" => res.push(T![_]),

11
crates/parser/src/grammar.rs
View file

@ -39,6 +39,7 @@ mod generic_params;
mod types;
use crate::{
grammar::expressions::FLOAT_LITERAL_FIRST,
parser::{CompletedMarker, Marker, Parser},
SyntaxKind::{self, *},
TokenSet, T,
@ -318,9 +319,15 @@ fn name_ref(p: &mut Parser) {
}
fn name_ref_or_index(p: &mut Parser) {
assert!(p.at(IDENT) || p.at(INT_NUMBER));
assert!(
p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST)
);
let m = p.start();
p.bump_any();
if p.at_ts(FLOAT_LITERAL_FIRST) {
p.bump_remap(FLOAT_NUMBER_PART);
} else {
p.bump_any();
}
m.complete(p, NAME_REF);
}

12
crates/parser/src/grammar/expressions.rs
View file

@ -3,7 +3,7 @@ mod atom;
use super::*;
pub(crate) use self::atom::{block_expr, match_arm_list};
pub(super) use self::atom::{literal, LITERAL_FIRST};
pub(super) use self::atom::{literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST};
#[derive(PartialEq, Eq)]
pub(super) enum Semicolon {
@ -452,6 +452,9 @@ fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
// fn foo() {
// x.foo();
// y.bar::<T>(1, 2,);
//
// 0e0.sin();
// 0e0f32.sin();
// }
fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
@ -469,17 +472,16 @@ fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
// fn foo() {
// x.foo;
// x.0.bar;
// x.0. bar;
// x.0.1;
// x.0();
// }
fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]));
let m = lhs.precede(p);
p.bump(T![.]);
if p.at(IDENT) || p.at(INT_NUMBER) {
if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) {
name_ref_or_index(p);
} else if p.at(FLOAT_NUMBER) {
// FIXME: How to recover and instead parse INT + T![.]?
p.bump_any();
} else {
p.error("expected field name or number");
}

40
crates/parser/src/grammar/expressions/atom.rs
View file

@ -17,22 +17,58 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[
T![true],
T![false],
INT_NUMBER,
FLOAT_NUMBER,
FLOAT_NUMBER_START_0,
FLOAT_NUMBER_START_1,
FLOAT_NUMBER_START_2,
BYTE,
CHAR,
STRING,
BYTE_STRING,
]);
pub(crate) const FLOAT_LITERAL_FIRST: TokenSet =
TokenSet::new(&[FLOAT_NUMBER_START_0, FLOAT_NUMBER_START_1, FLOAT_NUMBER_START_2]);
pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
if !p.at_ts(LITERAL_FIRST) {
return None;
}
let m = p.start();
p.bump_any();
if p.at_ts(FLOAT_LITERAL_FIRST) {
float_literal(p);
} else {
// Everything else is just one token.
p.bump_any();
}
Some(m.complete(p, LITERAL))
}
// test float_literal
// fn f() {
// 0.0;
// 1.;
// 0e0;
// 0e0f32;
// 1.23f64;
// }
pub(crate) fn float_literal(p: &mut Parser) {
// Floats can be up to 3 tokens. The first token indicates how many there are.
let f = p.start();
if p.at(FLOAT_NUMBER_START_0) {
p.bump(FLOAT_NUMBER_START_0);
} else if p.at(FLOAT_NUMBER_START_1) {
p.bump(FLOAT_NUMBER_START_1);
p.bump(DOT);
} else if p.at(FLOAT_NUMBER_START_2) {
p.bump(FLOAT_NUMBER_START_2);
p.bump(DOT);
p.bump(FLOAT_NUMBER_PART);
} else {
unreachable!();
}
f.complete(p, FLOAT_LITERAL);
}
// E.g. for after the break in `if break {}`, this should not match
pub(super) const ATOM_EXPR_FIRST: TokenSet =
LITERAL_FIRST.union(paths::PATH_FIRST).union(TokenSet::new(&[

2
crates/parser/src/grammar/patterns.rs
View file

@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> {
}
fn is_literal_pat_start(p: &Parser) -> bool {
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER)
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART)
|| p.at_ts(expressions::LITERAL_FIRST)
}

28
crates/parser/src/lexed_str.rs
View file

@ -177,7 +177,7 @@ impl<'a> Converter<'a> {
rustc_lexer::TokenKind::RawIdent => IDENT,
rustc_lexer::TokenKind::Literal { kind, .. } => {
self.extend_literal(token_text.len(), kind);
self.extend_literal(token_text, kind);
return;
}
@ -223,7 +223,7 @@ impl<'a> Converter<'a> {
self.push(syntax_kind, token_text.len(), err);
}
fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) {
fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) {
let mut err = "";
let syntax_kind = match *kind {
@ -237,7 +237,27 @@ impl<'a> Converter<'a> {
if empty_exponent {
err = "Missing digits after the exponent symbol";
}
FLOAT_NUMBER
// In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0`
// is lexed as a float, we split floats that contain a `.` into 3 tokens.
// To ensure that later stages can always reconstruct the token correctly, the first
// token in the sequence indicates the number of following tokens that are part of
// the float literal.
if let Some((before, after)) = token_text.split_once('.') {
let err = if err.is_empty() { None } else { Some(err) };
assert!(!before.is_empty());
let tok =
if after.is_empty() { FLOAT_NUMBER_START_1 } else { FLOAT_NUMBER_START_2 };
self.push(tok, before.len(), None);
self.push(DOT, 1, None);
if !after.is_empty() {
self.push(FLOAT_NUMBER_PART, after.len(), err);
}
return;
}
FLOAT_NUMBER_START_0
}
rustc_lexer::LiteralKind::Char { terminated } => {
if !terminated {
@ -295,6 +315,6 @@ impl<'a> Converter<'a> {
};
let err = if err.is_empty() { None } else { Some(err) };
self.push(syntax_kind, len, err);
self.push(syntax_kind, token_text.len(), err);
}
}

11
crates/parser/src/syntax_kind/generated.rs
View file

File diff suppressed because one or more lines are too long

44
crates/parser/test_data/lexer/err/empty_exponent.rast
View file

@ -1,14 +1,14 @@
FLOAT_NUMBER "0e" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_0 "0e" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER "0E" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_0 "0E" error: Missing digits after the exponent symbol
WHITESPACE "\n\n"
FLOAT_NUMBER "42e+" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_0 "42e+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER "42e-" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_0 "42e-" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER "42E+" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_0 "42E+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER "42E-" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_0 "42E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n"
INT_NUMBER "42"
DOT "."
@ -30,19 +30,35 @@ DOT "."
IDENT "E"
MINUS "-"
WHITESPACE "\n\n"
FLOAT_NUMBER "42.2e+" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER "42.2e-" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER "42.2E+" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER "42.2E-" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n"
FLOAT_NUMBER "42.2e+f32" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER "42.2e-f32" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER "42.2E+f32" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"
FLOAT_NUMBER "42.2E-f32" error: Missing digits after the exponent symbol
FLOAT_NUMBER_START_2 "42"
DOT "."
FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n"

25
crates/parser/test_data/lexer/ok/numbers.rast
View file

@ -4,7 +4,8 @@ INT_NUMBER "00"
WHITESPACE " "
INT_NUMBER "0_"
WHITESPACE " "
FLOAT_NUMBER "0."
FLOAT_NUMBER_START_1 "0"
DOT "."
WHITESPACE " "
INT_NUMBER "0z"
WHITESPACE "\n"
@ -20,11 +21,13 @@ INT_NUMBER "001279"
WHITESPACE " "
INT_NUMBER "0_1279"
WHITESPACE " "
FLOAT_NUMBER "0.1279"
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "1279"
WHITESPACE " "
FLOAT_NUMBER "0e1279"
FLOAT_NUMBER_START_0 "0e1279"
WHITESPACE " "
FLOAT_NUMBER "0E1279"
FLOAT_NUMBER_START_0 "0E1279"
WHITESPACE "\n"
INT_NUMBER "0"
DOT "."
@ -37,7 +40,7 @@ IDENT "foo"
L_PAREN "("
R_PAREN ")"
WHITESPACE "\n"
FLOAT_NUMBER "0e+1"
FLOAT_NUMBER_START_0 "0e+1"
WHITESPACE "\n"
INT_NUMBER "0"
DOT "."
@ -45,13 +48,19 @@ IDENT "e"
PLUS "+"
INT_NUMBER "1"
WHITESPACE "\n"
FLOAT_NUMBER "0.0E-2"
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "0E-2"
WHITESPACE "\n"
FLOAT_NUMBER "0___0.10000____0000e+111__"
FLOAT_NUMBER_START_2 "0___0"
DOT "."
FLOAT_NUMBER_PART "10000____0000e+111__"
WHITESPACE "\n"
INT_NUMBER "1i64"
WHITESPACE " "
FLOAT_NUMBER "92.0f32"
FLOAT_NUMBER_START_2 "92"
DOT "."
FLOAT_NUMBER_PART "0f32"
WHITESPACE " "
INT_NUMBER "11__s"
WHITESPACE "\n"

4
crates/parser/test_data/parser/err/0023_mismatched_paren.rast
View file

@ -32,7 +32,9 @@ SOURCE_FILE
INT_NUMBER "1"
COMMA ","
WHITESPACE " "
FLOAT_NUMBER "2.0"
FLOAT_NUMBER_START_2 "2"
DOT "."
FLOAT_NUMBER_PART "0"
WHITESPACE "\n "
R_CURLY "}"
WHITESPACE " "

33
crates/parser/test_data/parser/inline/ok/0011_field_expr.rast
View file

@ -40,6 +40,39 @@ SOURCE_FILE
IDENT "bar"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "0"
DOT "."
WHITESPACE " "
NAME_REF
IDENT "bar"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
FIELD_EXPR
FIELD_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "x"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "0"
DOT "."
NAME_REF
FLOAT_NUMBER_PART "1"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
CALL_EXPR
FIELD_EXPR

2
crates/parser/test_data/parser/inline/ok/0011_field_expr.rs
View file

@ -1,5 +1,7 @@
fn foo() {
x.foo;
x.0.bar;
x.0. bar;
x.0.1;
x.0();
}

5
crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast
View file

@ -57,7 +57,10 @@ SOURCE_FILE
EQ "="
WHITESPACE " "
LITERAL
FLOAT_NUMBER "2.0"
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "2"
DOT "."
FLOAT_NUMBER_PART "0"
SEMICOLON ";"
WHITESPACE "\n "
LET_STMT

26
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast
View file

@ -58,6 +58,32 @@ SOURCE_FILE
COMMA ","
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n\n "
EXPR_STMT
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0"
DOT "."
NAME_REF
IDENT "sin"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
METHOD_CALL_EXPR
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0f32"
DOT "."
NAME_REF
IDENT "sin"
ARG_LIST
L_PAREN "("
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"

3
crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs
View file

@ -1,4 +1,7 @@
fn foo() {
x.foo();
y.bar::<T>(1, 2,);
0e0.sin();
0e0f32.sin();
}

51
crates/parser/test_data/parser/inline/ok/0201_float_literal.rast Normal file
View file

@ -0,0 +1,51 @@
SOURCE_FILE
FN
FN_KW "fn"
WHITESPACE " "
NAME
IDENT "f"
PARAM_LIST
L_PAREN "("
R_PAREN ")"
WHITESPACE " "
BLOCK_EXPR
STMT_LIST
L_CURLY "{"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "0"
DOT "."
FLOAT_NUMBER_PART "0"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_1 "1"
DOT "."
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_0 "0e0f32"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
LITERAL
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "23f64"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"

7
crates/parser/test_data/parser/inline/ok/0201_float_literal.rs Normal file
View file

@ -0,0 +1,7 @@
fn f() {
0.0;
1.;
0e0;
0e0f32;
1.23f64;
}

10
crates/parser/test_data/parser/ok/0056_neq_in_type.rast
View file

@ -19,7 +19,10 @@ SOURCE_FILE
CAST_EXPR
METHOD_CALL_EXPR
LITERAL
FLOAT_NUMBER "1.0f32"
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "0f32"
DOT "."
NAME_REF
IDENT "floor"
@ -40,7 +43,10 @@ SOURCE_FILE
CAST_EXPR
METHOD_CALL_EXPR
LITERAL
FLOAT_NUMBER "1.0f32"
FLOAT_LITERAL
FLOAT_NUMBER_START_2 "1"
DOT "."
FLOAT_NUMBER_PART "0f32"
DOT "."
NAME_REF
IDENT "floor"

9
crates/syntax/rust.ungram
View file

@ -365,13 +365,20 @@ MacroExpr =
Literal =
Attr* value:(
'int_number' | 'float_number'
'int_number' | FloatLiteral
| 'string' | 'raw_string'
| 'byte_string' | 'raw_byte_string'
| 'true' | 'false'
| 'char' | 'byte'
)
FloatLiteral =
'float_number_start_0'?
'float_number_start_1'?
'float_number_start_2'?
'.'?
'float_number_part'?
PathExpr =
Attr* Path

42
crates/syntax/src/ast/expr_ext.rs
View file

@ -8,7 +8,7 @@ use crate::{
operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp},
support, AstChildren, AstNode,
},
AstToken,
AstToken, SyntaxElement,
SyntaxKind::*,
SyntaxNode, SyntaxToken, T,
};
@ -282,30 +282,32 @@ pub enum LiteralKind {
String(ast::String),
ByteString(ast::ByteString),
IntNumber(ast::IntNumber),
FloatNumber(ast::FloatNumber),
FloatNumber(ast::FloatLiteral),
Char(ast::Char),
Byte(ast::Byte),
Bool(bool),
}
impl ast::Literal {
pub fn token(&self) -> SyntaxToken {
pub fn value(&self) -> SyntaxElement {
self.syntax()
.children_with_tokens()
.find(|e| e.kind() != ATTR && !e.kind().is_trivia())
.and_then(|e| e.into_token())
.unwrap()
}
pub fn kind(&self) -> LiteralKind {
let token = self.token();
let token = match self.value() {
rowan::NodeOrToken::Node(node) => {
return LiteralKind::FloatNumber(
ast::FloatLiteral::cast(node).expect("unreachable"),
);
}
rowan::NodeOrToken::Token(token) => token,
};
if let Some(t) = ast::IntNumber::cast(token.clone()) {
return LiteralKind::IntNumber(t);
}
if let Some(t) = ast::FloatNumber::cast(token.clone()) {
return LiteralKind::FloatNumber(t);
}
if let Some(t) = ast::String::cast(token.clone()) {
return LiteralKind::String(t);
}
@ -325,6 +327,26 @@ impl ast::Literal {
_ => unreachable!(),
}
}
pub fn as_string(&self) -> Option<ast::String> {
match self.kind() {
LiteralKind::String(it) => Some(it),
_ => None,
}
}
pub fn as_byte_string(&self) -> Option<ast::ByteString> {
match self.kind() {
LiteralKind::ByteString(it) => Some(it),
_ => None,
}
}
}
impl ast::FloatLiteral {
pub fn suffix(&self) -> Option<String> {
ast::FloatNumberPart::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string())
}
}
pub enum BlockModifier {
@ -364,7 +386,7 @@ impl ast::BlockExpr {
fn test_literal_with_attr() {
let parse = ast::SourceFile::parse(r#"const _: &str = { #[attr] "Hello" };"#);
let lit = parse.tree().syntax().descendants().find_map(ast::Literal::cast).unwrap();
assert_eq!(lit.token().text(), r#""Hello""#);
assert_eq!(lit.value().to_string(), r#""Hello""#);
}
impl ast::RecordExprField {

36
crates/syntax/src/ast/generated/nodes.rs
View file

@ -1085,6 +1085,26 @@ impl UnderscoreExpr {
pub fn underscore_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![_]) }
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct FloatLiteral {
pub(crate) syntax: SyntaxNode,
}
impl FloatLiteral {
pub fn float_number_start_0_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_0])
}
pub fn float_number_start_1_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_1])
}
pub fn float_number_start_2_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_start_2])
}
pub fn dot_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![.]) }
pub fn float_number_part_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number_part])
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct StmtList {
pub(crate) syntax: SyntaxNode,
@ -2719,6 +2739,17 @@ impl AstNode for UnderscoreExpr {
}
fn syntax(&self) -> &SyntaxNode { &self.syntax }
}
impl AstNode for FloatLiteral {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_LITERAL }
fn cast(syntax: SyntaxNode) -> Option<Self> {
if Self::can_cast(syntax.kind()) {
Some(Self { syntax })
} else {
None
}
}
fn syntax(&self) -> &SyntaxNode { &self.syntax }
}
impl AstNode for StmtList {
fn can_cast(kind: SyntaxKind) -> bool { kind == STMT_LIST }
fn cast(syntax: SyntaxNode) -> Option<Self> {
@ -4608,6 +4639,11 @@ impl std::fmt::Display for UnderscoreExpr {
std::fmt::Display::fmt(self.syntax(), f)
}
}
impl std::fmt::Display for FloatLiteral {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f)
}
}
impl std::fmt::Display for StmtList {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self.syntax(), f)

8
crates/syntax/src/ast/generated/tokens.rs
View file

@ -112,16 +112,16 @@ impl AstToken for IntNumber {
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct FloatNumber {
pub struct FloatNumberPart {
pub(crate) syntax: SyntaxToken,
}
impl std::fmt::Display for FloatNumber {
impl std::fmt::Display for FloatNumberPart {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.syntax, f)
}
}
impl AstToken for FloatNumber {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER }
impl AstToken for FloatNumberPart {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER_PART }
fn cast(syntax: SyntaxToken) -> Option<Self> {
if Self::can_cast(syntax.kind()) {
Some(Self { syntax })

13
crates/syntax/src/ast/make.rs
View file

@ -799,6 +799,11 @@ pub fn struct_(
))
}
pub fn literal(text: &str) -> ast::Literal {
assert_eq!(text.trim(), text);
ast_from_text(&format!("fn f() {{ let _ = {}; }}", text))
}
#[track_caller]
fn ast_from_text<N: AstNode>(text: &str) -> N {
let parse = SourceFile::parse(text);
@ -827,7 +832,7 @@ pub fn token(kind: SyntaxKind) -> SyntaxToken {
pub mod tokens {
use once_cell::sync::Lazy;
use crate::{ast, AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken};
use crate::{AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken};
pub(super) static SOURCE_FILE: Lazy<Parse<SourceFile>> = Lazy::new(|| {
SourceFile::parse(
@ -858,12 +863,6 @@ pub mod tokens {
sf.syntax().first_child_or_token().unwrap().into_token().unwrap()
}
pub fn literal(text: &str) -> SyntaxToken {
assert_eq!(text.trim(), text);
let lit: ast::Literal = super::ast_from_text(&format!("fn f() {{ let _ = {}; }}", text));
lit.syntax().first_child_or_token().unwrap().into_token().unwrap()
}
pub fn single_newline() -> SyntaxToken {
let res = SOURCE_FILE
.tree()

4
crates/syntax/src/ast/node_ext.rs
View file

@ -555,7 +555,9 @@ impl ast::FieldExpr {
self.syntax
.children_with_tokens()
// FIXME: Accepting floats here to reject them in validation later
.find(|c| c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER)
.find(|c| {
c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER_PART
})
.as_ref()
.and_then(SyntaxElement::as_token)
.cloned()

29
crates/syntax/src/ast/token_ext.rs
View file

@ -321,7 +321,7 @@ impl ast::IntNumber {
}
}
impl ast::FloatNumber {
impl ast::FloatNumberPart {
pub fn suffix(&self) -> Option<&str> {
let text = self.text();
let mut indices = text.char_indices();
@ -355,14 +355,24 @@ impl Radix {
#[cfg(test)]
mod tests {
use crate::ast::{self, make, FloatNumber, IntNumber};
use crate::ast::{self, make};
fn check_float_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
assert_eq!(FloatNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into());
let suffix = match make::literal(lit).kind() {
ast::LiteralKind::FloatNumber(f) => f.suffix(),
// `1f32` lexes as an INT_NUMBER
ast::LiteralKind::IntNumber(i) => i.suffix().map(|s| s.to_string()),
e => unreachable!("{e:?}"),
};
assert_eq!(suffix.as_deref(), expected.into());
}
fn check_int_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
assert_eq!(IntNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into());
let i = match make::literal(lit).kind() {
ast::LiteralKind::IntNumber(i) => i,
_ => unreachable!(),
};
assert_eq!(i.suffix(), expected.into());
}
#[test]
@ -390,12 +400,11 @@ mod tests {
}
fn check_string_value<'a>(lit: &str, expected: impl Into<Option<&'a str>>) {
assert_eq!(
ast::String { syntax: make::tokens::literal(&format!("\"{}\"", lit)) }
.value()
.as_deref(),
expected.into()
);
let s = match make::literal(&format!("\"{}\"", lit)).kind() {
ast::LiteralKind::String(s) => s,
_ => unreachable!(),
};
assert_eq!(s.value().as_deref(), expected.into());
}
#[test]

13
crates/syntax/src/tests/ast_src.rs
View file

@ -71,7 +71,17 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
"super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield",
],
contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"],
literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"],
literals: &[
"INT_NUMBER",
"FLOAT_NUMBER_START_0",
"FLOAT_NUMBER_START_1",
"FLOAT_NUMBER_START_2",
"FLOAT_NUMBER_PART",
"CHAR",
"BYTE",
"STRING",
"BYTE_STRING",
],
tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"],
nodes: &[
"SOURCE_FILE",
@ -183,6 +193,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
"PATH",
"PATH_SEGMENT",
"LITERAL",
"FLOAT_LITERAL",
"RENAME",
"VISIBILITY",
"WHERE_CLAUSE",

6
crates/syntax/src/tests/sourcegen_ast.rs
View file

@ -462,6 +462,10 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String {
[lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT };
[ident] => { $crate::SyntaxKind::IDENT };
[shebang] => { $crate::SyntaxKind::SHEBANG };
[float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART };
[float_number_start_0] => { $crate::SyntaxKind::FLOAT_NUMBER_START_0 };
[float_number_start_1] => { $crate::SyntaxKind::FLOAT_NUMBER_START_1 };
[float_number_start_2] => { $crate::SyntaxKind::FLOAT_NUMBER_START_2 };
}
pub use T;
};
@ -585,7 +589,7 @@ impl Field {
fn lower(grammar: &Grammar) -> AstSrc {
let mut res = AstSrc {
tokens: "Whitespace Comment String ByteString IntNumber FloatNumber Char Byte Ident"
tokens: "Whitespace Comment String ByteString IntNumber FloatNumberPart Char Byte Ident"
.split_ascii_whitespace()
.map(|it| it.to_string())
.collect::<Vec<_>>(),

11
crates/syntax/src/validation.rs
View file

@ -119,8 +119,15 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
text.rfind(end_delimiter).and_then(|end| text.get(prefix_len..end))
}
let token = literal.token();
let text = token.text();
let token = literal.value();
let text;
let text = match &token {
rowan::NodeOrToken::Node(node) => {
text = node.text().to_string();
&*text
}
rowan::NodeOrToken::Token(token) => token.text(),
};
// FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205)
let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| {