mirror of
https://github.com/rust-lang/rust-analyzer
synced 2024-12-26 13:03:31 +00:00
internal: Bump rustc_lexer
This commit is contained in:
parent
0a806fe7ad
commit
099b5b3b15
26 changed files with 134 additions and 73 deletions
64
Cargo.lock
generated
64
Cargo.lock
generated
|
@ -1193,7 +1193,7 @@ dependencies = [
|
|||
"drop_bomb",
|
||||
"expect-test",
|
||||
"limit",
|
||||
"rustc-ap-rustc_lexer",
|
||||
"ra-ap-rustc_lexer",
|
||||
"sourcegen",
|
||||
"stdx",
|
||||
]
|
||||
|
@ -1396,6 +1396,16 @@ dependencies = [
|
|||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ra-ap-rustc_lexer"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1c145702ed3f237918e512685185dc8a4d0edc3a5326c63d20361d8ba9b45b3"
|
||||
dependencies = [
|
||||
"unic-emoji-char",
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.7.0"
|
||||
|
@ -1524,15 +1534,6 @@ dependencies = [
|
|||
"xshell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-ap-rustc_lexer"
|
||||
version = "727.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f40f26e7abdcd3b982f36c09a634cc6187988fbf6ec466c91f8d30a12ac0237"
|
||||
dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.22"
|
||||
|
@ -1753,9 +1754,9 @@ dependencies = [
|
|||
"proc-macro2",
|
||||
"profile",
|
||||
"quote",
|
||||
"ra-ap-rustc_lexer",
|
||||
"rayon",
|
||||
"rowan",
|
||||
"rustc-ap-rustc_lexer",
|
||||
"rustc-hash",
|
||||
"smol_str",
|
||||
"sourcegen",
|
||||
|
@ -1998,6 +1999,47 @@ version = "1.16.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3e5df347f0bf3ec1d670aad6ca5c6a1859cd9ea61d2113125794654ccced68f"
|
||||
|
||||
[[package]]
|
||||
name = "unic-char-property"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221"
|
||||
dependencies = [
|
||||
"unic-char-range",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unic-char-range"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc"
|
||||
|
||||
[[package]]
|
||||
name = "unic-common"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
|
||||
|
||||
[[package]]
|
||||
name = "unic-emoji-char"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d"
|
||||
dependencies = [
|
||||
"unic-char-property",
|
||||
"unic-char-range",
|
||||
"unic-ucd-version",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unic-ucd-version"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4"
|
||||
dependencies = [
|
||||
"unic-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicase"
|
||||
version = "2.6.0"
|
||||
|
|
|
@ -89,3 +89,5 @@ text-size = "1.1.0"
|
|||
serde = { version = "=1.0.156", features = ["derive"] }
|
||||
serde_json = "1.0.94"
|
||||
triomphe = { version = "0.1.8", default-features = false, features = ["std"] }
|
||||
|
||||
rustc_lexer = { version = "0.1.0", package = "ra-ap-rustc_lexer" }
|
||||
|
|
|
@ -13,7 +13,7 @@ doctest = false
|
|||
|
||||
[dependencies]
|
||||
drop_bomb = "0.1.5"
|
||||
rustc_lexer = { version = "727.0.0", package = "rustc-ap-rustc_lexer" }
|
||||
rustc_lexer.workspace = true
|
||||
|
||||
limit.workspace = true
|
||||
|
||||
|
|
|
@ -36,7 +36,7 @@ impl<'a> LexedStr<'a> {
|
|||
};
|
||||
|
||||
for token in rustc_lexer::tokenize(&text[conv.offset..]) {
|
||||
let token_text = &text[conv.offset..][..token.len];
|
||||
let token_text = &text[conv.offset..][..token.len as usize];
|
||||
|
||||
conv.extend_token(&token.kind, token_text);
|
||||
}
|
||||
|
@ -49,8 +49,8 @@ impl<'a> LexedStr<'a> {
|
|||
return None;
|
||||
}
|
||||
|
||||
let token = rustc_lexer::first_token(text);
|
||||
if token.len != text.len() {
|
||||
let token = rustc_lexer::tokenize(text).next()?;
|
||||
if token.len as usize != text.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
|
@ -175,6 +175,10 @@ impl<'a> Converter<'a> {
|
|||
rustc_lexer::TokenKind::Ident => {
|
||||
SyntaxKind::from_keyword(token_text).unwrap_or(IDENT)
|
||||
}
|
||||
rustc_lexer::TokenKind::InvalidIdent => {
|
||||
err = "Ident contains invalid characters";
|
||||
IDENT
|
||||
}
|
||||
|
||||
rustc_lexer::TokenKind::RawIdent => IDENT,
|
||||
rustc_lexer::TokenKind::Literal { kind, .. } => {
|
||||
|
@ -221,6 +225,7 @@ impl<'a> Converter<'a> {
|
|||
err = "unknown literal prefix";
|
||||
IDENT
|
||||
}
|
||||
rustc_lexer::TokenKind::Eof => EOF,
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -268,35 +273,30 @@ impl<'a> Converter<'a> {
|
|||
}
|
||||
BYTE_STRING
|
||||
}
|
||||
rustc_lexer::LiteralKind::RawStr { err: raw_str_err, .. } => {
|
||||
if let Some(raw_str_err) = raw_str_err {
|
||||
err = match raw_str_err {
|
||||
rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw string literal",
|
||||
rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found {
|
||||
"Missing trailing `\"` to terminate the raw string literal"
|
||||
} else {
|
||||
"Missing trailing `\"` with `#` symbols to terminate the raw string literal"
|
||||
},
|
||||
rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols",
|
||||
};
|
||||
};
|
||||
rustc_lexer::LiteralKind::CStr { terminated } => {
|
||||
if !terminated {
|
||||
err = "Missing trailing `\"` symbol to terminate the string literal";
|
||||
}
|
||||
STRING
|
||||
}
|
||||
rustc_lexer::LiteralKind::RawByteStr { err: raw_str_err, .. } => {
|
||||
if let Some(raw_str_err) = raw_str_err {
|
||||
err = match raw_str_err {
|
||||
rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw byte string literal",
|
||||
rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found {
|
||||
"Missing trailing `\"` to terminate the raw byte string literal"
|
||||
} else {
|
||||
"Missing trailing `\"` with `#` symbols to terminate the raw byte string literal"
|
||||
},
|
||||
rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols",
|
||||
};
|
||||
};
|
||||
|
||||
rustc_lexer::LiteralKind::RawStr { n_hashes } => {
|
||||
if n_hashes.is_none() {
|
||||
err = "Invalid raw string literal";
|
||||
}
|
||||
STRING
|
||||
}
|
||||
rustc_lexer::LiteralKind::RawByteStr { n_hashes } => {
|
||||
if n_hashes.is_none() {
|
||||
err = "Invalid raw string literal";
|
||||
}
|
||||
BYTE_STRING
|
||||
}
|
||||
rustc_lexer::LiteralKind::RawCStr { n_hashes } => {
|
||||
if n_hashes.is_none() {
|
||||
err = "Invalid raw string literal";
|
||||
}
|
||||
STRING
|
||||
}
|
||||
};
|
||||
|
||||
let err = if err.is_empty() { None } else { Some(err) };
|
||||
|
|
|
@ -117,6 +117,7 @@ pub enum SyntaxKind {
|
|||
BYTE,
|
||||
STRING,
|
||||
BYTE_STRING,
|
||||
C_STRING,
|
||||
ERROR,
|
||||
IDENT,
|
||||
WHITESPACE,
|
||||
|
@ -379,7 +380,7 @@ impl SyntaxKind {
|
|||
)
|
||||
}
|
||||
pub fn is_literal(self) -> bool {
|
||||
matches!(self, INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING)
|
||||
matches!(self, INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING | C_STRING)
|
||||
}
|
||||
pub fn from_keyword(ident: &str) -> Option<SyntaxKind> {
|
||||
let kw = match ident {
|
||||
|
|
|
@ -1 +1 @@
|
|||
BYTE_STRING "br##\"" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
|
||||
BYTE_STRING "br##\"" error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
BYTE_STRING "br##\"\\x7f" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
|
||||
BYTE_STRING "br##\"\\x7f" error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
BYTE_STRING "br##\"🦀" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
|
||||
BYTE_STRING "br##\"🦀" error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
BYTE_STRING "br##\"\\" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
|
||||
BYTE_STRING "br##\"\\" error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
BYTE_STRING "br##\"\\n" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
|
||||
BYTE_STRING "br##\"\\n" error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
BYTE_STRING "br##\" " error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
|
||||
BYTE_STRING "br##\" " error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
BYTE_STRING "br##\"\\u{20AA}" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
|
||||
BYTE_STRING "br##\"\\u{20AA}" error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
STRING "r##\"" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
|
||||
STRING "r##\"" error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
STRING "r##\"\\x7f" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
|
||||
STRING "r##\"\\x7f" error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
STRING "r##\"🦀" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
|
||||
STRING "r##\"🦀" error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
STRING "r##\"\\" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
|
||||
STRING "r##\"\\" error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
STRING "r##\"\\n" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
|
||||
STRING "r##\"\\n" error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
STRING "r##\" " error: Missing trailing `"` with `#` symbols to terminate the raw string literal
|
||||
STRING "r##\" " error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
STRING "r##\"\\u{20AA}" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
|
||||
STRING "r##\"\\u{20AA}" error: Invalid raw string literal
|
||||
|
|
|
@ -1 +1 @@
|
|||
BYTE_STRING "br##" error: Missing `"` symbol after `#` symbols to begin the raw byte string literal
|
||||
BYTE_STRING "br##" error: Invalid raw string literal
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
BYTE_STRING "br## " error: Missing `"` symbol after `#` symbols to begin the raw byte string literal
|
||||
BYTE_STRING "br## " error: Invalid raw string literal
|
||||
IDENT "I"
|
||||
WHITESPACE " "
|
||||
IDENT "lack"
|
||||
|
|
|
@ -1 +1 @@
|
|||
STRING "r##" error: Missing `"` symbol after `#` symbols to begin the raw string literal
|
||||
STRING "r##" error: Invalid raw string literal
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
STRING "r## " error: Missing `"` symbol after `#` symbols to begin the raw string literal
|
||||
STRING "r## " error: Invalid raw string literal
|
||||
IDENT "I"
|
||||
WHITESPACE " "
|
||||
IDENT "lack"
|
||||
|
|
|
@ -17,13 +17,14 @@ cov-mark = "2.0.0-pre.1"
|
|||
either = "1.7.0"
|
||||
itertools = "0.10.5"
|
||||
rowan = "0.15.11"
|
||||
rustc_lexer = { version = "727.0.0", package = "rustc-ap-rustc_lexer" }
|
||||
rustc-hash = "1.1.0"
|
||||
once_cell = "1.17.0"
|
||||
indexmap = "1.9.1"
|
||||
smol_str.workspace = true
|
||||
triomphe.workspace = true
|
||||
|
||||
rustc_lexer.workspace = true
|
||||
|
||||
parser.workspace = true
|
||||
profile.workspace = true
|
||||
stdx.workspace = true
|
||||
|
|
|
@ -71,7 +71,7 @@ pub(crate) const KINDS_SRC: KindsSrc<'_> = KindsSrc {
|
|||
"super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield",
|
||||
],
|
||||
contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules", "yeet"],
|
||||
literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"],
|
||||
literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING", "C_STRING"],
|
||||
tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"],
|
||||
nodes: &[
|
||||
"SOURCE_FILE",
|
||||
|
|
|
@ -5,7 +5,7 @@
|
|||
mod block;
|
||||
|
||||
use rowan::Direction;
|
||||
use rustc_lexer::unescape::{self, unescape_byte, unescape_char, unescape_literal, Mode};
|
||||
use rustc_lexer::unescape::{self, unescape_literal, Mode};
|
||||
|
||||
use crate::{
|
||||
algo,
|
||||
|
@ -44,7 +44,7 @@ pub(crate) fn validate(root: &SyntaxNode) -> Vec<SyntaxError> {
|
|||
errors
|
||||
}
|
||||
|
||||
fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str {
|
||||
fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> (&'static str, bool) {
|
||||
use unescape::EscapeError as EE;
|
||||
|
||||
#[rustfmt::skip]
|
||||
|
@ -103,12 +103,15 @@ fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str {
|
|||
EE::UnicodeEscapeInByte => {
|
||||
"Byte literals must not contain unicode escapes"
|
||||
}
|
||||
EE::NonAsciiCharInByte | EE::NonAsciiCharInByteString => {
|
||||
EE::NonAsciiCharInByte => {
|
||||
"Byte literals must not contain non-ASCII characters"
|
||||
}
|
||||
EE::UnskippedWhitespaceWarning => "Whitespace after this escape is not skipped",
|
||||
EE::MultipleSkippedLinesWarning => "Multiple lines are skipped by this escape",
|
||||
|
||||
};
|
||||
|
||||
err_message
|
||||
(err_message, err.is_fatal())
|
||||
}
|
||||
|
||||
fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
|
||||
|
@ -121,9 +124,13 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
|
|||
let text = token.text();
|
||||
|
||||
// FIXME: lift this lambda refactor to `fn` (https://github.com/rust-lang/rust-analyzer/pull/2834#discussion_r366199205)
|
||||
let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| {
|
||||
let mut push_err = |prefix_len, off, err: unescape::EscapeError| {
|
||||
let off = token.text_range().start() + TextSize::try_from(off + prefix_len).unwrap();
|
||||
acc.push(SyntaxError::new_at_offset(rustc_unescape_error_to_string(err), off));
|
||||
let (message, is_err) = rustc_unescape_error_to_string(err);
|
||||
// FIXME: Emit lexer warnings
|
||||
if is_err {
|
||||
acc.push(SyntaxError::new_at_offset(message, off));
|
||||
}
|
||||
};
|
||||
|
||||
match literal.kind() {
|
||||
|
@ -132,7 +139,7 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
|
|||
if let Some(without_quotes) = unquote(text, 1, '"') {
|
||||
unescape_literal(without_quotes, Mode::Str, &mut |range, char| {
|
||||
if let Err(err) = char {
|
||||
push_err(1, (range.start, err));
|
||||
push_err(1, range.start, err);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
@ -143,20 +150,28 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
|
|||
if let Some(without_quotes) = unquote(text, 2, '"') {
|
||||
unescape_literal(without_quotes, Mode::ByteStr, &mut |range, char| {
|
||||
if let Err(err) = char {
|
||||
push_err(2, (range.start, err));
|
||||
push_err(1, range.start, err);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
ast::LiteralKind::Char(_) => {
|
||||
if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape_char) {
|
||||
push_err(1, e);
|
||||
if let Some(without_quotes) = unquote(text, 1, '\'') {
|
||||
unescape_literal(without_quotes, Mode::Char, &mut |range, char| {
|
||||
if let Err(err) = char {
|
||||
push_err(1, range.start, err);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
ast::LiteralKind::Byte(_) => {
|
||||
if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape_byte) {
|
||||
push_err(2, e);
|
||||
if let Some(without_quotes) = unquote(text, 2, '\'') {
|
||||
unescape_literal(without_quotes, Mode::Byte, &mut |range, char| {
|
||||
if let Err(err) = char {
|
||||
push_err(2, range.start, err);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
ast::LiteralKind::IntNumber(_)
|
||||
|
|
Loading…
Reference in a new issue