Mirrors/rust-analyzer
2
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer synced 2024-12-27 05:23:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

Split float literal tokens at the .

Browse source
This commit is contained in:
Jonas Schievink 2022-05-04 16:51:05 +02:00
parent 502c519e7d
commit 1bc3305d95
23 changed files with 157 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
crates/ide/src/syntax_highlighting/highlight.rs
View file

@ -30,7 +30,7 @@ pub(super) fn token(sema: &Semantics<RootDatabase>, token: SyntaxToken) -> Optio
INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => { INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => {
SymbolKind::Field.into() SymbolKind::Field.into()
} }
INT_NUMBER | FLOAT_NUMBER => HlTag::NumericLiteral.into(), INT_NUMBER | FLOAT_NUMBER_PART => HlTag::NumericLiteral.into(),
BYTE => HlTag::ByteLiteral.into(), BYTE => HlTag::ByteLiteral.into(),
CHAR => HlTag::CharLiteral.into(), CHAR => HlTag::CharLiteral.into(),
IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => { IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => {

14
crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
View file

@ -119,13 +119,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">-</span><span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">-</span><span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">#</span><span class="numeric_literal">0</span><span class="numeric_literal">10</span><span class="variable">x</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">27</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">#</span><span class="numeric_literal">0</span><span class="numeric_literal">10</span><span class="variable">x</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">27</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="operator">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="operator">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="operator">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="operator">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="operator">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0</span><span class="operator">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234.56</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234</span><span class="operator">.</span><span class="numeric_literal">56</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">&gt;</span><span class="numeric_literal">8</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 right-aligned characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">&gt;</span><span class="numeric_literal">8</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 right-aligned characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello {{}}"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello {{}}"</span><span class="parenthesis">)</span><span class="semicolon">;</span>

29
crates/mbe/src/syntax_bridge.rs
View file

@ -260,6 +260,35 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree {
IDENT => make_leaf!(Ident), IDENT => make_leaf!(Ident),
UNDERSCORE => make_leaf!(Ident), UNDERSCORE => make_leaf!(Ident),
k if k.is_keyword() => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident),
FLOAT_NUMBER_PART => {
// Reassemble a split-up float token.
let mut range = range;
let mut text = token.to_text(conv).to_string();
if let Some(dot) = conv.peek() {
if dot.kind(conv) == DOT {
let (_, dot_range) = conv.bump().unwrap();
text += &*dot.to_text(conv);
range = TextRange::new(range.start(), dot_range.end());
if let Some(tail) = conv.peek() {
if tail.kind(conv) == FLOAT_NUMBER_PART {
let (_, tail_range) = conv.bump().unwrap();
text += &*tail.to_text(conv);
range = TextRange::new(range.start(), tail_range.end());
}
}
}
}
result.push(
tt::Leaf::from(tt::Literal {
id: conv.id_alloc().alloc(range, synth_id),
text: text.into(),
})
.into(),
);
continue;
}
k if k.is_literal() => make_leaf!(Literal), k if k.is_literal() => make_leaf!(Literal),
LIFETIME_IDENT => { LIFETIME_IDENT => {
let char_unit = TextSize::of('\''); let char_unit = TextSize::of('\'');

16
crates/mbe/src/to_parser_input.rs
View file

@ -35,15 +35,13 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input {
let is_negated = lit.text.starts_with('-'); let is_negated = lit.text.starts_with('-');
let inner_text = &lit.text[if is_negated { 1 } else { 0 }..]; let inner_text = &lit.text[if is_negated { 1 } else { 0 }..];
let kind = parser::LexedStr::single_token(inner_text) let lexed_str = parser::LexedStr::new(inner_text);
.map(|(kind, _error)| kind) if lexed_str.is_empty() {
.filter(|kind| { panic!("failed to convert literal: {:?}", lit);
kind.is_literal() }
&& (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER)) for i in 0..lexed_str.len() {
}) res.push(lexed_str.kind(i));
.unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit)); }
res.push(kind);
} }
tt::Leaf::Ident(ident) => match ident.text.as_ref() { tt::Leaf::Ident(ident) => match ident.text.as_ref() {
"_" => res.push(T![_]), "_" => res.push(T![_]),

2
crates/parser/src/grammar.rs
View file

@ -318,7 +318,7 @@ fn name_ref(p: &mut Parser) {
} }
fn name_ref_or_index(p: &mut Parser) { fn name_ref_or_index(p: &mut Parser) {
assert!(p.at(IDENT) || p.at(INT_NUMBER)); assert!(p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART));
let m = p.start(); let m = p.start();
p.bump_any(); p.bump_any();
m.complete(p, NAME_REF); m.complete(p, NAME_REF);

5
crates/parser/src/grammar/expressions.rs
View file

@ -475,11 +475,8 @@ fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.])); assert!(p.at(T![.]));
let m = lhs.precede(p); let m = lhs.precede(p);
p.bump(T![.]); p.bump(T![.]);
if p.at(IDENT) || p.at(INT_NUMBER) { if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) {
name_ref_or_index(p); name_ref_or_index(p);
} else if p.at(FLOAT_NUMBER) {
// FIXME: How to recover and instead parse INT + T![.]?
p.bump_any();
} else { } else {
p.error("expected field name or number"); p.error("expected field name or number");
} }

14
crates/parser/src/grammar/expressions/atom.rs
View file

@ -17,7 +17,7 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[
T![true], T![true],
T![false], T![false],
INT_NUMBER, INT_NUMBER,
FLOAT_NUMBER, FLOAT_NUMBER_PART,
BYTE, BYTE,
CHAR, CHAR,
STRING, STRING,
@ -29,11 +29,19 @@ pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
return None; return None;
} }
let m = p.start(); let m = p.start();
if p.at(FLOAT_NUMBER) { if p.at(FLOAT_NUMBER_PART) {
// Floats can be up to 3 tokens: 2 `FLOAT_NUMBER_PART`s separated by 1 `DOT`
let f = p.start(); let f = p.start();
p.bump(FLOAT_NUMBER); p.bump(FLOAT_NUMBER_PART);
if p.at(DOT) {
p.bump(DOT);
if p.at(FLOAT_NUMBER_PART) {
p.bump(FLOAT_NUMBER_PART);
}
}
f.complete(p, FLOAT_LITERAL); f.complete(p, FLOAT_LITERAL);
} else { } else {
// Everything else is just one token.
p.bump_any(); p.bump_any();
} }
Some(m.complete(p, LITERAL)) Some(m.complete(p, LITERAL))

2
crates/parser/src/grammar/patterns.rs
View file

@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> {
} }
fn is_literal_pat_start(p: &Parser) -> bool { fn is_literal_pat_start(p: &Parser) -> bool {
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER) p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART)
|| p.at_ts(expressions::LITERAL_FIRST) || p.at_ts(expressions::LITERAL_FIRST)
} }

23
crates/parser/src/lexed_str.rs
View file

@ -177,7 +177,7 @@ impl<'a> Converter<'a> {
rustc_lexer::TokenKind::RawIdent => IDENT, rustc_lexer::TokenKind::RawIdent => IDENT,
rustc_lexer::TokenKind::Literal { kind, .. } => { rustc_lexer::TokenKind::Literal { kind, .. } => {
self.extend_literal(token_text.len(), kind); self.extend_literal(token_text, kind);
return; return;
} }
@ -223,7 +223,7 @@ impl<'a> Converter<'a> {
self.push(syntax_kind, token_text.len(), err); self.push(syntax_kind, token_text.len(), err);
} }
fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) { fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) {
let mut err = ""; let mut err = "";
let syntax_kind = match *kind { let syntax_kind = match *kind {
@ -237,7 +237,22 @@ impl<'a> Converter<'a> {
if empty_exponent { if empty_exponent {
err = "Missing digits after the exponent symbol"; err = "Missing digits after the exponent symbol";
} }
FLOAT_NUMBER
// In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0`
// is lexed as a float, we split floats that contain a `.` into 3 tokens.
if let Some((before, after)) = token_text.split_once('.') {
let err = if err.is_empty() { None } else { Some(err) };
if !before.is_empty() {
self.push(FLOAT_NUMBER_PART, before.len(), None);
}
self.push(DOT, 1, None);
if !after.is_empty() {
self.push(FLOAT_NUMBER_PART, after.len(), err);
}
return;
}
FLOAT_NUMBER_PART
} }
rustc_lexer::LiteralKind::Char { terminated } => { rustc_lexer::LiteralKind::Char { terminated } => {
if !terminated { if !terminated {
@ -295,6 +310,6 @@ impl<'a> Converter<'a> {
}; };
let err = if err.is_empty() { None } else { Some(err) }; let err = if err.is_empty() { None } else { Some(err) };
self.push(syntax_kind, len, err); self.push(syntax_kind, token_text.len(), err);
} }
} }

6
crates/parser/src/syntax_kind/generated.rs
View file

File diff suppressed because one or more lines are too long

44
crates/parser/test_data/lexer/err/empty_exponent.rast
View file

@ -1,14 +1,14 @@
FLOAT_NUMBER "0e" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "0e" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "0E" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "0E" error: Missing digits after the exponent symbol
WHITESPACE "\n\n" WHITESPACE "\n\n"
FLOAT_NUMBER "42e+" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "42e+" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "42e-" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "42e-" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "42E+" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "42E+" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "42E-" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "42E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n" WHITESPACE "\n\n"
INT_NUMBER "42" INT_NUMBER "42"
DOT "." DOT "."
@ -30,19 +30,35 @@ DOT "."
IDENT "E" IDENT "E"
MINUS "-" MINUS "-"
WHITESPACE "\n\n" WHITESPACE "\n\n"
FLOAT_NUMBER "42.2e+" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "42"
DOT "."
FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "42.2e-" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "42"
DOT "."
FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "42.2E+" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "42"
DOT "."
FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "42.2E-" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "42"
DOT "."
FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol
WHITESPACE "\n\n" WHITESPACE "\n\n"
FLOAT_NUMBER "42.2e+f32" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "42"
DOT "."
FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "42.2e-f32" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "42"
DOT "."
FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "42.2E+f32" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "42"
DOT "."
FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "42.2E-f32" error: Missing digits after the exponent symbol FLOAT_NUMBER_PART "42"
DOT "."
FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol
WHITESPACE "\n" WHITESPACE "\n"

25
crates/parser/test_data/lexer/ok/numbers.rast
View file

@ -4,7 +4,8 @@ INT_NUMBER "00"
WHITESPACE " " WHITESPACE " "
INT_NUMBER "0_" INT_NUMBER "0_"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER "0." FLOAT_NUMBER_PART "0"
DOT "."
WHITESPACE " " WHITESPACE " "
INT_NUMBER "0z" INT_NUMBER "0z"
WHITESPACE "\n" WHITESPACE "\n"
@ -20,11 +21,13 @@ INT_NUMBER "001279"
WHITESPACE " " WHITESPACE " "
INT_NUMBER "0_1279" INT_NUMBER "0_1279"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER "0.1279" FLOAT_NUMBER_PART "0"
DOT "."
FLOAT_NUMBER_PART "1279"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER "0e1279" FLOAT_NUMBER_PART "0e1279"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER "0E1279" FLOAT_NUMBER_PART "0E1279"
WHITESPACE "\n" WHITESPACE "\n"
INT_NUMBER "0" INT_NUMBER "0"
DOT "." DOT "."
@ -37,7 +40,7 @@ IDENT "foo"
L_PAREN "(" L_PAREN "("
R_PAREN ")" R_PAREN ")"
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "0e+1" FLOAT_NUMBER_PART "0e+1"
WHITESPACE "\n" WHITESPACE "\n"
INT_NUMBER "0" INT_NUMBER "0"
DOT "." DOT "."
@ -45,13 +48,19 @@ IDENT "e"
PLUS "+" PLUS "+"
INT_NUMBER "1" INT_NUMBER "1"
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "0.0E-2" FLOAT_NUMBER_PART "0"
DOT "."
FLOAT_NUMBER_PART "0E-2"
WHITESPACE "\n" WHITESPACE "\n"
FLOAT_NUMBER "0___0.10000____0000e+111__" FLOAT_NUMBER_PART "0___0"
DOT "."
FLOAT_NUMBER_PART "10000____0000e+111__"
WHITESPACE "\n" WHITESPACE "\n"
INT_NUMBER "1i64" INT_NUMBER "1i64"
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER "92.0f32" FLOAT_NUMBER_PART "92"
DOT "."
FLOAT_NUMBER_PART "0f32"
WHITESPACE " " WHITESPACE " "
INT_NUMBER "11__s" INT_NUMBER "11__s"
WHITESPACE "\n" WHITESPACE "\n"

4
crates/parser/test_data/parser/err/0023_mismatched_paren.rast
View file

@ -32,7 +32,9 @@ SOURCE_FILE
INT_NUMBER "1" INT_NUMBER "1"
COMMA "," COMMA ","
WHITESPACE " " WHITESPACE " "
FLOAT_NUMBER "2.0" FLOAT_NUMBER_PART "2"
DOT "."
FLOAT_NUMBER_PART "0"
WHITESPACE "\n " WHITESPACE "\n "
R_CURLY "}" R_CURLY "}"
WHITESPACE " " WHITESPACE " "

4
crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast
View file

@ -58,7 +58,9 @@ SOURCE_FILE
WHITESPACE " " WHITESPACE " "
LITERAL LITERAL
FLOAT_LITERAL FLOAT_LITERAL
FLOAT_NUMBER "2.0" FLOAT_NUMBER_PART "2"
DOT "."
FLOAT_NUMBER_PART "0"
SEMICOLON ";" SEMICOLON ";"
WHITESPACE "\n " WHITESPACE "\n "
LET_STMT LET_STMT

8
crates/parser/test_data/parser/ok/0056_neq_in_type.rast
View file

@ -20,7 +20,9 @@ SOURCE_FILE
METHOD_CALL_EXPR METHOD_CALL_EXPR
LITERAL LITERAL
FLOAT_LITERAL FLOAT_LITERAL
FLOAT_NUMBER "1.0f32" FLOAT_NUMBER_PART "1"
DOT "."
FLOAT_NUMBER_PART "0f32"
DOT "." DOT "."
NAME_REF NAME_REF
IDENT "floor" IDENT "floor"
@ -42,7 +44,9 @@ SOURCE_FILE
METHOD_CALL_EXPR METHOD_CALL_EXPR
LITERAL LITERAL
FLOAT_LITERAL FLOAT_LITERAL
FLOAT_NUMBER "1.0f32" FLOAT_NUMBER_PART "1"
DOT "."
FLOAT_NUMBER_PART "0f32"
DOT "." DOT "."
NAME_REF NAME_REF
IDENT "floor" IDENT "floor"

4
crates/syntax/rust.ungram
View file

@ -373,7 +373,9 @@ Literal =
) )
FloatLiteral = FloatLiteral =
'float_number' 'float_number_part'
'.'?
'float_number_part'?
PathExpr = PathExpr =
Attr* Path Attr* Path

2
crates/syntax/src/ast/expr_ext.rs
View file

@ -345,7 +345,7 @@ impl ast::Literal {
impl ast::FloatLiteral { impl ast::FloatLiteral {
pub fn suffix(&self) -> Option<String> { pub fn suffix(&self) -> Option<String> {
ast::FloatNumber::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string()) ast::FloatNumberPart::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string())
} }
} }

5
crates/syntax/src/ast/generated/nodes.rs
View file

@ -1090,9 +1090,10 @@ pub struct FloatLiteral {
pub(crate) syntax: SyntaxNode, pub(crate) syntax: SyntaxNode,
} }
impl FloatLiteral { impl FloatLiteral {
pub fn float_number_token(&self) -> Option<SyntaxToken> { pub fn float_number_part_token(&self) -> Option<SyntaxToken> {
support::token(&self.syntax, T![float_number]) support::token(&self.syntax, T![float_number_part])
} }
pub fn dot_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![.]) }
} }
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]

8
crates/syntax/src/ast/generated/tokens.rs
View file

@ -112,16 +112,16 @@ impl AstToken for IntNumber {
} }
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct FloatNumber { pub struct FloatNumberPart {
pub(crate) syntax: SyntaxToken, pub(crate) syntax: SyntaxToken,
} }
impl std::fmt::Display for FloatNumber { impl std::fmt::Display for FloatNumberPart {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.syntax, f) std::fmt::Display::fmt(&self.syntax, f)
} }
} }
impl AstToken for FloatNumber { impl AstToken for FloatNumberPart {
fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER } fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER_PART }
fn cast(syntax: SyntaxToken) -> Option<Self> { fn cast(syntax: SyntaxToken) -> Option<Self> {
if Self::can_cast(syntax.kind()) { if Self::can_cast(syntax.kind()) {
Some(Self { syntax }) Some(Self { syntax })

4
crates/syntax/src/ast/node_ext.rs
View file

@ -555,7 +555,9 @@ impl ast::FieldExpr {
self.syntax self.syntax
.children_with_tokens() .children_with_tokens()
// FIXME: Accepting floats here to reject them in validation later // FIXME: Accepting floats here to reject them in validation later
.find(|c| c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER) .find(|c| {
c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER_PART
})
.as_ref() .as_ref()
.and_then(SyntaxElement::as_token) .and_then(SyntaxElement::as_token)
.cloned() .cloned()

2
crates/syntax/src/ast/token_ext.rs
View file

@ -321,7 +321,7 @@ impl ast::IntNumber {
} }
} }
impl ast::FloatNumber { impl ast::FloatNumberPart {
pub fn suffix(&self) -> Option<&str> { pub fn suffix(&self) -> Option<&str> {
let text = self.text(); let text = self.text();
let mut indices = text.char_indices(); let mut indices = text.char_indices();

2
crates/syntax/src/tests/ast_src.rs
View file

@ -71,7 +71,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
"super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield", "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield",
], ],
contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"], contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"],
literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"], literals: &["INT_NUMBER", "FLOAT_NUMBER_PART", "CHAR", "BYTE", "STRING", "BYTE_STRING"],
tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"], tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"],
nodes: &[ nodes: &[
"SOURCE_FILE", "SOURCE_FILE",

4
crates/syntax/src/tests/sourcegen_ast.rs
View file

@ -462,7 +462,7 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String {
[lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT }; [lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT };
[ident] => { $crate::SyntaxKind::IDENT }; [ident] => { $crate::SyntaxKind::IDENT };
[shebang] => { $crate::SyntaxKind::SHEBANG }; [shebang] => { $crate::SyntaxKind::SHEBANG };
[float_number] => { $crate::SyntaxKind::FLOAT_NUMBER }; [float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART };
} }
pub use T; pub use T;
}; };
@ -586,7 +586,7 @@ impl Field {
fn lower(grammar: &Grammar) -> AstSrc { fn lower(grammar: &Grammar) -> AstSrc {
let mut res = AstSrc { let mut res = AstSrc {
tokens: "Whitespace Comment String ByteString IntNumber FloatNumber Char Byte Ident" tokens: "Whitespace Comment String ByteString IntNumber FloatNumberPart Char Byte Ident"
.split_ascii_whitespace() .split_ascii_whitespace()
.map(|it| it.to_string()) .map(|it| it.to_string())
.collect::<Vec<_>>(), .collect::<Vec<_>>(),