diff --git a/crates/ra_hir/src/macros.rs b/crates/ra_hir/src/macros.rs index 9ba3816856..c23ad53cc4 100644 --- a/crates/ra_hir/src/macros.rs +++ b/crates/ra_hir/src/macros.rs @@ -221,12 +221,24 @@ fn convert_tt(tt: &SyntaxNode) -> Option { if child == first_child || child == last_child || child.kind().is_trivia() { continue; } - let child = if child.kind() == TOKEN_TREE { + let child: tt::TokenTree = if child.kind() == TOKEN_TREE { convert_tt(child)?.into() - } else if child.kind().is_keyword() { + } else if child.kind().is_keyword() || child.kind() == IDENT { let text = child.leaf_text().unwrap().clone(); tt::Leaf::from(tt::Ident { text }).into() + } else if child.kind().is_punct() { + // FIXME: multibyte tokens + tt::Leaf::from(tt::Punct { + char: child.text().char_at(0)?, + }) + .into() + } else if child.kind().is_literal() { + tt::Leaf::from(tt::Literal { + text: child.leaf_text().unwrap().clone(), + }) + .into() } else { + log::error!("unknown kind: {:?}", child); return None; }; token_trees.push(child) diff --git a/crates/ra_syntax/src/grammar.ron b/crates/ra_syntax/src/grammar.ron index d4c8637054..47334bdf08 100644 --- a/crates/ra_syntax/src/grammar.ron +++ b/crates/ra_syntax/src/grammar.ron @@ -24,6 +24,7 @@ Grammar( ["/", "SLASH"], ["^", "CARET"], ["%", "PERCENT"], + ["_", "UNDERSCORE"], ], // Tokens for which the longest match must be chosen (e.g. `..` is a DOTDOT, but `.` is a DOT) multi_byte_tokens: [ @@ -99,20 +100,21 @@ Grammar( "default", "union", ], - tokens: [ - "ERROR", - "IDENT", - "UNDERSCORE", - "WHITESPACE", + literals: [ "INT_NUMBER", "FLOAT_NUMBER", - "LIFETIME", "CHAR", "BYTE", "STRING", "RAW_STRING", "BYTE_STRING", "RAW_BYTE_STRING", + ], + tokens: [ + "ERROR", + "IDENT", + "WHITESPACE", + "LIFETIME", "COMMENT", "SHEBANG", ], diff --git a/crates/ra_syntax/src/syntax_kinds/generated.rs b/crates/ra_syntax/src/syntax_kinds/generated.rs index 14f610891c..fea5134584 100644 --- a/crates/ra_syntax/src/syntax_kinds/generated.rs +++ b/crates/ra_syntax/src/syntax_kinds/generated.rs @@ -36,6 +36,7 @@ pub enum SyntaxKind { SLASH, CARET, PERCENT, + UNDERSCORE, DOT, DOTDOT, DOTDOTDOT, @@ -103,19 +104,18 @@ pub enum SyntaxKind { AUTO_KW, DEFAULT_KW, UNION_KW, - ERROR, - IDENT, - UNDERSCORE, - WHITESPACE, INT_NUMBER, FLOAT_NUMBER, - LIFETIME, CHAR, BYTE, STRING, RAW_STRING, BYTE_STRING, RAW_BYTE_STRING, + ERROR, + IDENT, + WHITESPACE, + LIFETIME, COMMENT, SHEBANG, SOURCE_FILE, @@ -275,6 +275,79 @@ impl SyntaxKind { } } + pub fn is_punct(self) -> bool { + match self { + | SEMI + | COMMA + | L_PAREN + | R_PAREN + | L_CURLY + | R_CURLY + | L_BRACK + | R_BRACK + | L_ANGLE + | R_ANGLE + | AT + | POUND + | TILDE + | QUESTION + | DOLLAR + | AMP + | PIPE + | PLUS + | STAR + | SLASH + | CARET + | PERCENT + | UNDERSCORE + | DOT + | DOTDOT + | DOTDOTDOT + | DOTDOTEQ + | COLON + | COLONCOLON + | EQ + | EQEQ + | FAT_ARROW + | EXCL + | NEQ + | MINUS + | THIN_ARROW + | LTEQ + | GTEQ + | PLUSEQ + | MINUSEQ + | PIPEEQ + | AMPEQ + | CARETEQ + | SLASHEQ + | STAREQ + | PERCENTEQ + | AMPAMP + | PIPEPIPE + | SHL + | SHR + | SHLEQ + | SHREQ + => true, + _ => false + } + } + pub fn is_literal(self) -> bool { + match self { + | INT_NUMBER + | FLOAT_NUMBER + | CHAR + | BYTE + | STRING + | RAW_STRING + | BYTE_STRING + | RAW_BYTE_STRING + => true, + _ => false + } + } + pub(crate) fn info(self) -> &'static SyntaxInfo { match self { SEMI => &SyntaxInfo { name: "SEMI" }, @@ -299,6 +372,7 @@ impl SyntaxKind { SLASH => &SyntaxInfo { name: "SLASH" }, CARET => &SyntaxInfo { name: "CARET" }, PERCENT => &SyntaxInfo { name: "PERCENT" }, + UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" }, DOT => &SyntaxInfo { name: "DOT" }, DOTDOT => &SyntaxInfo { name: "DOTDOT" }, DOTDOTDOT => &SyntaxInfo { name: "DOTDOTDOT" }, @@ -366,19 +440,18 @@ impl SyntaxKind { AUTO_KW => &SyntaxInfo { name: "AUTO_KW" }, DEFAULT_KW => &SyntaxInfo { name: "DEFAULT_KW" }, UNION_KW => &SyntaxInfo { name: "UNION_KW" }, - ERROR => &SyntaxInfo { name: "ERROR" }, - IDENT => &SyntaxInfo { name: "IDENT" }, - UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" }, - WHITESPACE => &SyntaxInfo { name: "WHITESPACE" }, INT_NUMBER => &SyntaxInfo { name: "INT_NUMBER" }, FLOAT_NUMBER => &SyntaxInfo { name: "FLOAT_NUMBER" }, - LIFETIME => &SyntaxInfo { name: "LIFETIME" }, CHAR => &SyntaxInfo { name: "CHAR" }, BYTE => &SyntaxInfo { name: "BYTE" }, STRING => &SyntaxInfo { name: "STRING" }, RAW_STRING => &SyntaxInfo { name: "RAW_STRING" }, BYTE_STRING => &SyntaxInfo { name: "BYTE_STRING" }, RAW_BYTE_STRING => &SyntaxInfo { name: "RAW_BYTE_STRING" }, + ERROR => &SyntaxInfo { name: "ERROR" }, + IDENT => &SyntaxInfo { name: "IDENT" }, + WHITESPACE => &SyntaxInfo { name: "WHITESPACE" }, + LIFETIME => &SyntaxInfo { name: "LIFETIME" }, COMMENT => &SyntaxInfo { name: "COMMENT" }, SHEBANG => &SyntaxInfo { name: "SHEBANG" }, SOURCE_FILE => &SyntaxInfo { name: "SOURCE_FILE" }, @@ -559,6 +632,7 @@ impl SyntaxKind { '/' => SLASH, '^' => CARET, '%' => PERCENT, + '_' => UNDERSCORE, _ => return None, }; Some(tok) diff --git a/crates/ra_syntax/src/syntax_kinds/generated.rs.tera b/crates/ra_syntax/src/syntax_kinds/generated.rs.tera index 21f9444b1e..837437136d 100644 --- a/crates/ra_syntax/src/syntax_kinds/generated.rs.tera +++ b/crates/ra_syntax/src/syntax_kinds/generated.rs.tera @@ -23,7 +23,7 @@ pub enum SyntaxKind { {% for kw in concat(a=keywords, b=contextual_keywords) %} {{kw | upper}}_KW, {%- endfor -%} -{% for t in concat(a=tokens, b=nodes) %} +{% for t in concat(a=literals, b=tokens, c=nodes) %} {{t}}, {%- endfor %} } @@ -40,6 +40,25 @@ impl SyntaxKind { } } + pub fn is_punct(self) -> bool { + match self { +{%- for t in concat(a=single_byte_tokens, b=multi_byte_tokens) %} + | {{t.1}} +{%- endfor %} + => true, + _ => false + } + } + pub fn is_literal(self) -> bool { + match self { +{%- for t in literals %} + | {{t}} +{%- endfor %} + => true, + _ => false + } + } + pub(crate) fn info(self) -> &'static SyntaxInfo { match self { {%- for t in concat(a=single_byte_tokens, b=multi_byte_tokens) %} @@ -48,7 +67,7 @@ impl SyntaxKind { {% for kw in concat(a=keywords, b=contextual_keywords) %} {{kw | upper}}_KW => &SyntaxInfo { name: "{{kw | upper}}_KW" }, {%- endfor -%} -{% for t in concat(a=tokens, b=nodes) %} +{% for t in concat(a=literals, b=tokens, c=nodes) %} {{t}} => &SyntaxInfo { name: "{{t}}" }, {%- endfor %} TOMBSTONE => &SyntaxInfo { name: "TOMBSTONE" },