From 89779ca55e42d8c148659014906de05f5a82e519 Mon Sep 17 00:00:00 2001 From: davidsemakula Date: Tue, 9 Apr 2024 14:32:46 +0300 Subject: [PATCH 1/4] internal: improve `TokenSet` implementation --- crates/parser/src/token_set.rs | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/crates/parser/src/token_set.rs b/crates/parser/src/token_set.rs index cd4894c1e8..2d615b7a5d 100644 --- a/crates/parser/src/token_set.rs +++ b/crates/parser/src/token_set.rs @@ -4,32 +4,46 @@ use crate::SyntaxKind; /// A bit-set of `SyntaxKind`s #[derive(Clone, Copy)] -pub(crate) struct TokenSet(u128); +pub(crate) struct TokenSet([u64; 3]); + +const LAST_TOKEN_KIND_DISCRIMINANT: usize = SyntaxKind::SHEBANG as usize; impl TokenSet { - pub(crate) const EMPTY: TokenSet = TokenSet(0); + pub(crate) const EMPTY: TokenSet = TokenSet([0; 3]); pub(crate) const fn new(kinds: &[SyntaxKind]) -> TokenSet { - let mut res = 0u128; + let mut res = [0; 3]; let mut i = 0; while i < kinds.len() { - res |= mask(kinds[i]); + let kind = kinds[i]; + debug_assert!( + kind as usize <= LAST_TOKEN_KIND_DISCRIMINANT, + "Expected a token `SyntaxKind`" + ); + let idx = kind as usize / 64; + res[idx] |= mask(kind); i += 1; } TokenSet(res) } pub(crate) const fn union(self, other: TokenSet) -> TokenSet { - TokenSet(self.0 | other.0) + TokenSet([self.0[0] | other.0[0], self.0[1] | other.0[1], self.0[2] | other.0[2]]) } pub(crate) const fn contains(&self, kind: SyntaxKind) -> bool { - self.0 & mask(kind) != 0 + debug_assert!( + kind as usize <= LAST_TOKEN_KIND_DISCRIMINANT, + "Expected a token `SyntaxKind`" + ); + let idx = kind as usize / 64; + self.0[idx] & mask(kind) != 0 } } -const fn mask(kind: SyntaxKind) -> u128 { - 1u128 << (kind as usize) +const fn mask(kind: SyntaxKind) -> u64 { + debug_assert!(kind as usize <= LAST_TOKEN_KIND_DISCRIMINANT, "Expected a token `SyntaxKind`"); + 1 << (kind as usize % 64) } #[test] From e326b634d1d63955c9e9bf2d442ebf0f4740dc34 Mon Sep 17 00:00:00 2001 From: davidsemakula Date: Tue, 9 Apr 2024 18:05:54 +0300 Subject: [PATCH 2/4] internal: add reserved keywords --- crates/parser/src/syntax_kind/generated.rs | 31 ++++++++++++++++++---- xtask/src/codegen/grammar/ast_src.rs | 10 +++---- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs index 6ecfdc9f46..ef83420c52 100644 --- a/crates/parser/src/syntax_kind/generated.rs +++ b/crates/parser/src/syntax_kind/generated.rs @@ -61,9 +61,11 @@ pub enum SyntaxKind { SHR, SHLEQ, SHREQ, + ABSTRACT_KW, AS_KW, ASYNC_KW, AWAIT_KW, + BECOME_KW, BOX_KW, BREAK_KW, CONST_KW, @@ -75,6 +77,7 @@ pub enum SyntaxKind { ENUM_KW, EXTERN_KW, FALSE_KW, + FINAL_KW, FN_KW, FOR_KW, IF_KW, @@ -87,10 +90,11 @@ pub enum SyntaxKind { MOD_KW, MOVE_KW, MUT_KW, + OVERRIDE_KW, + PRIV_KW, PUB_KW, REF_KW, RETURN_KW, - BECOME_KW, SELF_KW, SELF_TYPE_KW, STATIC_KW, @@ -100,8 +104,11 @@ pub enum SyntaxKind { TRUE_KW, TRY_KW, TYPE_KW, + TYPEOF_KW, UNSAFE_KW, + UNSIZED_KW, USE_KW, + VIRTUAL_KW, WHERE_KW, WHILE_KW, YIELD_KW, @@ -280,9 +287,11 @@ impl SyntaxKind { pub fn is_keyword(self) -> bool { matches!( self, - AS_KW + ABSTRACT_KW + | AS_KW | ASYNC_KW | AWAIT_KW + | BECOME_KW | BOX_KW | BREAK_KW | CONST_KW @@ -294,6 +303,7 @@ impl SyntaxKind { | ENUM_KW | EXTERN_KW | FALSE_KW + | FINAL_KW | FN_KW | FOR_KW | IF_KW @@ -306,10 +316,11 @@ impl SyntaxKind { | MOD_KW | MOVE_KW | MUT_KW + | OVERRIDE_KW + | PRIV_KW | PUB_KW | REF_KW | RETURN_KW - | BECOME_KW | SELF_KW | SELF_TYPE_KW | STATIC_KW @@ -319,8 +330,11 @@ impl SyntaxKind { | TRUE_KW | TRY_KW | TYPE_KW + | TYPEOF_KW | UNSAFE_KW + | UNSIZED_KW | USE_KW + | VIRTUAL_KW | WHERE_KW | WHILE_KW | YIELD_KW @@ -399,9 +413,11 @@ impl SyntaxKind { } pub fn from_keyword(ident: &str) -> Option { let kw = match ident { + "abstract" => ABSTRACT_KW, "as" => AS_KW, "async" => ASYNC_KW, "await" => AWAIT_KW, + "become" => BECOME_KW, "box" => BOX_KW, "break" => BREAK_KW, "const" => CONST_KW, @@ -413,6 +429,7 @@ impl SyntaxKind { "enum" => ENUM_KW, "extern" => EXTERN_KW, "false" => FALSE_KW, + "final" => FINAL_KW, "fn" => FN_KW, "for" => FOR_KW, "if" => IF_KW, @@ -425,10 +442,11 @@ impl SyntaxKind { "mod" => MOD_KW, "move" => MOVE_KW, "mut" => MUT_KW, + "override" => OVERRIDE_KW, + "priv" => PRIV_KW, "pub" => PUB_KW, "ref" => REF_KW, "return" => RETURN_KW, - "become" => BECOME_KW, "self" => SELF_KW, "Self" => SELF_TYPE_KW, "static" => STATIC_KW, @@ -438,8 +456,11 @@ impl SyntaxKind { "true" => TRUE_KW, "try" => TRY_KW, "type" => TYPE_KW, + "typeof" => TYPEOF_KW, "unsafe" => UNSAFE_KW, + "unsized" => UNSIZED_KW, "use" => USE_KW, + "virtual" => VIRTUAL_KW, "where" => WHERE_KW, "while" => WHILE_KW, "yield" => YIELD_KW, @@ -500,4 +521,4 @@ impl SyntaxKind { } } #[macro_export] -macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [do] => { $ crate :: SyntaxKind :: DO_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [become] => { $ crate :: SyntaxKind :: BECOME_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [builtin] => { $ crate :: SyntaxKind :: BUILTIN_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [yeet] => { $ crate :: SyntaxKind :: YEET_KW } ; [offset_of] => { $ crate :: SyntaxKind :: OFFSET_OF_KW } ; [asm] => { $ crate :: SyntaxKind :: ASM_KW } ; [format_args] => { $ crate :: SyntaxKind :: FORMAT_ARGS_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; } +macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [abstract] => { $ crate :: SyntaxKind :: ABSTRACT_KW } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [become] => { $ crate :: SyntaxKind :: BECOME_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [do] => { $ crate :: SyntaxKind :: DO_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [final] => { $ crate :: SyntaxKind :: FINAL_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [override] => { $ crate :: SyntaxKind :: OVERRIDE_KW } ; [priv] => { $ crate :: SyntaxKind :: PRIV_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [typeof] => { $ crate :: SyntaxKind :: TYPEOF_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [unsized] => { $ crate :: SyntaxKind :: UNSIZED_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [virtual] => { $ crate :: SyntaxKind :: VIRTUAL_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [builtin] => { $ crate :: SyntaxKind :: BUILTIN_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [yeet] => { $ crate :: SyntaxKind :: YEET_KW } ; [offset_of] => { $ crate :: SyntaxKind :: OFFSET_OF_KW } ; [asm] => { $ crate :: SyntaxKind :: ASM_KW } ; [format_args] => { $ crate :: SyntaxKind :: FORMAT_ARGS_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; } diff --git a/xtask/src/codegen/grammar/ast_src.rs b/xtask/src/codegen/grammar/ast_src.rs index 8221c57789..c246ee9950 100644 --- a/xtask/src/codegen/grammar/ast_src.rs +++ b/xtask/src/codegen/grammar/ast_src.rs @@ -65,11 +65,11 @@ pub(crate) const KINDS_SRC: KindsSrc<'_> = KindsSrc { (">>=", "SHREQ"), ], keywords: &[ - "as", "async", "await", "box", "break", "const", "continue", "crate", "do", "dyn", "else", - "enum", "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop", "macro", - "match", "mod", "move", "mut", "pub", "ref", "return", "become", "self", "Self", "static", - "struct", "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", - "yield", + "abstract", "as", "async", "await", "become", "box", "break", "const", "continue", "crate", + "do", "dyn", "else", "enum", "extern", "false", "final", "fn", "for", "if", "impl", "in", + "let", "loop", "macro", "match", "mod", "move", "mut", "override", "priv", "pub", "ref", + "return", "self", "Self", "static", "struct", "super", "trait", "true", "try", "type", + "typeof", "unsafe", "unsized", "use", "virtual", "where", "while", "yield", ], contextual_keywords: &[ "auto", From 8e459125df5eebef6d7b07c5646ba6eefba63d05 Mon Sep 17 00:00:00 2001 From: davidsemakula Date: Tue, 9 Apr 2024 18:08:25 +0300 Subject: [PATCH 3/4] internal: add auto-import assist tests for raw identifiers --- .../ide-assists/src/handlers/auto_import.rs | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/crates/ide-assists/src/handlers/auto_import.rs b/crates/ide-assists/src/handlers/auto_import.rs index 5545e41f53..b90bccb48e 100644 --- a/crates/ide-assists/src/handlers/auto_import.rs +++ b/crates/ide-assists/src/handlers/auto_import.rs @@ -1620,4 +1620,50 @@ mod foo { "#, ); } + + #[test] + fn preserve_raw_identifiers_strict() { + check_assist( + auto_import, + r" + r#as$0 + + pub mod ffi_mod { + pub fn r#as() {}; + } + ", + r" + use ffi_mod::r#as; + + r#as + + pub mod ffi_mod { + pub fn r#as() {}; + } + ", + ); + } + + #[test] + fn preserve_raw_identifiers_reserved() { + check_assist( + auto_import, + r" + r#abstract$0 + + pub mod ffi_mod { + pub fn r#abstract() {}; + } + ", + r" + use ffi_mod::r#abstract; + + r#abstract + + pub mod ffi_mod { + pub fn r#abstract() {}; + } + ", + ); + } } From 69fe457cb540c1880e860e03a287f1313942f2cd Mon Sep 17 00:00:00 2001 From: davidsemakula Date: Mon, 15 Apr 2024 16:46:55 +0300 Subject: [PATCH 4/4] internal: simplify `TokenSet` implementation --- crates/parser/src/token_set.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/parser/src/token_set.rs b/crates/parser/src/token_set.rs index 2d615b7a5d..88a89a53a7 100644 --- a/crates/parser/src/token_set.rs +++ b/crates/parser/src/token_set.rs @@ -6,6 +6,9 @@ use crate::SyntaxKind; #[derive(Clone, Copy)] pub(crate) struct TokenSet([u64; 3]); +/// `TokenSet`s should only include token `SyntaxKind`s, so the discriminant of any passed/included +/// `SyntaxKind` must *not* be greater than that of the last token `SyntaxKind`. +/// See #17037. const LAST_TOKEN_KIND_DISCRIMINANT: usize = SyntaxKind::SHEBANG as usize; impl TokenSet { @@ -15,13 +18,13 @@ impl TokenSet { let mut res = [0; 3]; let mut i = 0; while i < kinds.len() { - let kind = kinds[i]; + let discriminant = kinds[i] as usize; debug_assert!( - kind as usize <= LAST_TOKEN_KIND_DISCRIMINANT, + discriminant <= LAST_TOKEN_KIND_DISCRIMINANT, "Expected a token `SyntaxKind`" ); - let idx = kind as usize / 64; - res[idx] |= mask(kind); + let idx = discriminant / 64; + res[idx] |= 1 << (discriminant % 64); i += 1; } TokenSet(res) @@ -32,20 +35,17 @@ impl TokenSet { } pub(crate) const fn contains(&self, kind: SyntaxKind) -> bool { + let discriminant = kind as usize; debug_assert!( - kind as usize <= LAST_TOKEN_KIND_DISCRIMINANT, + discriminant <= LAST_TOKEN_KIND_DISCRIMINANT, "Expected a token `SyntaxKind`" ); - let idx = kind as usize / 64; - self.0[idx] & mask(kind) != 0 + let idx = discriminant / 64; + let mask = 1 << (discriminant % 64); + self.0[idx] & mask != 0 } } -const fn mask(kind: SyntaxKind) -> u64 { - debug_assert!(kind as usize <= LAST_TOKEN_KIND_DISCRIMINANT, "Expected a token `SyntaxKind`"); - 1 << (kind as usize % 64) -} - #[test] fn token_set_works_for_tokens() { use crate::SyntaxKind::*;