generate single byte tokens

2024-12-26 13:03:31 +00:00 · 2018-07-30 17:46:50 +03:00 · 2018-07-30 17:46:50 +03:00 · 7fb7cfc2d1
commit 7fb7cfc2d1
parent d436694097
5 changed files with 139 additions and 103 deletions
--- a/src/grammar.ron
+++ b/src/grammar.ron
@ -1,4 +1,43 @@
 Grammar(
+    single_byte_tokens: [
+        [";", "SEMI"],
+        [",", "COMMA"],
+        ["(", "L_PAREN"],
+        [")", "R_PAREN"],
+        ["{", "L_CURLY"],
+        ["}", "R_CURLY"],
+        ["[", "L_BRACK"],
+        ["]", "R_BRACK"],
+        ["<", "L_ANGLE"],
+        [">", "R_ANGLE"],
+        ["@", "AT"],
+        ["#", "POUND"],
+        ["~", "TILDE"],
+        ["?", "QUESTION"],
+        ["$", "DOLLAR"],
+        ["&", "AMPERSAND"],
+        ["|", "PIPE"],
+        ["+", "PLUS"],
+        ["*", "STAR"],
+        ["/", "SLASH"],
+        ["^", "CARET"],
+        ["%", "PERCENT"],
+    ],
+    multi_byte_tokens: [
+        [".", "DOT"],
+        ["..", "DOTDOT"],
+        ["...", "DOTDOTDOT"],
+        ["..=", "DOTDOTEQ"],
+        [":", "COLON"],
+        ["::", "COLONCOLON"],
+        ["=", "EQ"],
+        ["==", "EQEQ"],
+        ["=>", "FAT_ARROW"],
+        ["!", "EXCL"],
+        ["!=", "NEQ"],
+        ["-", "MINUS"],
+        ["->", "THIN_ARROW"],
+    ],
    keywords: [
        "use",
        "fn",
@ -42,32 +81,6 @@ Grammar(
        "WHITESPACE",
        "INT_NUMBER",
        "FLOAT_NUMBER",
-        "SEMI",
-        "COMMA",
-        "DOT",
-        "DOTDOT",
-        "DOTDOTDOT",
-        "DOTDOTEQ",
-        "L_PAREN",
-        "R_PAREN",
-        "L_CURLY",
-        "R_CURLY",
-        "L_BRACK",
-        "R_BRACK",
-        "L_ANGLE",
-        "R_ANGLE",
-        "AT",
-        "POUND",
-        "TILDE",
-        "QUESTION",
-        "COLON",
-        "COLONCOLON",
-        "DOLLAR",
-        "EQ",
-        "EQEQ",
-        "FAT_ARROW",
-        "NEQ",
-        "EXCL",
        "LIFETIME",
        "CHAR",
        "BYTE",
@ -75,15 +88,6 @@ Grammar(
        "RAW_STRING",
        "BYTE_STRING",
        "RAW_BYTE_STRING",
-        "PLUS",
-        "MINUS",
-        "STAR",
-        "SLASH",
-        "CARET",
-        "PERCENT",
-        "AMPERSAND",
-        "PIPE",
-        "THIN_ARROW",
        "COMMENT",
        "DOC_COMMENT",
        "SHEBANG",
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@ -79,30 +79,11 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
    }

    // One-byte tokens.
-    match c {
-        ';' => return SEMI,
-        ',' => return COMMA,
-        '(' => return L_PAREN,
-        ')' => return R_PAREN,
-        '{' => return L_CURLY,
-        '}' => return R_CURLY,
-        '[' => return L_BRACK,
-        ']' => return R_BRACK,
-        '<' => return L_ANGLE,
-        '>' => return R_ANGLE,
-        '@' => return AT,
-        '#' => return POUND,
-        '~' => return TILDE,
-        '?' => return QUESTION,
-        '$' => return DOLLAR,
-        '&' => return AMPERSAND,
-        '|' => return PIPE,
-        '+' => return PLUS,
-        '*' => return STAR,
-        '/' => return SLASH,
-        '^' => return CARET,
-        '%' => return PERCENT,
+    if let Some(kind) = SyntaxKind::from_char(c) {
+        return kind;
+    }

+    match c {
        // Multi-byte tokens.
        '.' => {
            return match (ptr.next(), ptr.nnext()) {
--- a/src/syntax_kinds/generated.rs
+++ b/src/syntax_kinds/generated.rs
@ -5,18 +5,8 @@ use super::SyntaxInfo;
 /// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum SyntaxKind {
-    ERROR,
-    IDENT,
-    UNDERSCORE,
-    WHITESPACE,
-    INT_NUMBER,
-    FLOAT_NUMBER,
    SEMI,
    COMMA,
-    DOT,
-    DOTDOT,
-    DOTDOTDOT,
-    DOTDOTEQ,
    L_PAREN,
    R_PAREN,
    L_CURLY,
@ -29,14 +19,33 @@ pub enum SyntaxKind {
    POUND,
    TILDE,
    QUESTION,
+    DOLLAR,
+    AMPERSAND,
+    PIPE,
+    PLUS,
+    STAR,
+    SLASH,
+    CARET,
+    PERCENT,
+    DOT,
+    DOTDOT,
+    DOTDOTDOT,
+    DOTDOTEQ,
    COLON,
    COLONCOLON,
-    DOLLAR,
    EQ,
    EQEQ,
    FAT_ARROW,
-    NEQ,
    EXCL,
+    NEQ,
+    MINUS,
+    THIN_ARROW,
+    ERROR,
+    IDENT,
+    UNDERSCORE,
+    WHITESPACE,
+    INT_NUMBER,
+    FLOAT_NUMBER,
    LIFETIME,
    CHAR,
    BYTE,
@ -44,15 +53,6 @@ pub enum SyntaxKind {
    RAW_STRING,
    BYTE_STRING,
    RAW_BYTE_STRING,
-    PLUS,
-    MINUS,
-    STAR,
-    SLASH,
-    CARET,
-    PERCENT,
-    AMPERSAND,
-    PIPE,
-    THIN_ARROW,
    COMMENT,
    DOC_COMMENT,
    SHEBANG,
@ -151,18 +151,8 @@ use self::SyntaxKind::*;
 impl SyntaxKind {
    pub(crate) fn info(self) -> &'static SyntaxInfo {
        match self {
-            ERROR => &SyntaxInfo { name: "ERROR" },
-            IDENT => &SyntaxInfo { name: "IDENT" },
-            UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" },
-            WHITESPACE => &SyntaxInfo { name: "WHITESPACE" },
-            INT_NUMBER => &SyntaxInfo { name: "INT_NUMBER" },
-            FLOAT_NUMBER => &SyntaxInfo { name: "FLOAT_NUMBER" },
            SEMI => &SyntaxInfo { name: "SEMI" },
            COMMA => &SyntaxInfo { name: "COMMA" },
-            DOT => &SyntaxInfo { name: "DOT" },
-            DOTDOT => &SyntaxInfo { name: "DOTDOT" },
-            DOTDOTDOT => &SyntaxInfo { name: "DOTDOTDOT" },
-            DOTDOTEQ => &SyntaxInfo { name: "DOTDOTEQ" },
            L_PAREN => &SyntaxInfo { name: "L_PAREN" },
            R_PAREN => &SyntaxInfo { name: "R_PAREN" },
            L_CURLY => &SyntaxInfo { name: "L_CURLY" },
@ -175,14 +165,33 @@ impl SyntaxKind {
            POUND => &SyntaxInfo { name: "POUND" },
            TILDE => &SyntaxInfo { name: "TILDE" },
            QUESTION => &SyntaxInfo { name: "QUESTION" },
+            DOLLAR => &SyntaxInfo { name: "DOLLAR" },
+            AMPERSAND => &SyntaxInfo { name: "AMPERSAND" },
+            PIPE => &SyntaxInfo { name: "PIPE" },
+            PLUS => &SyntaxInfo { name: "PLUS" },
+            STAR => &SyntaxInfo { name: "STAR" },
+            SLASH => &SyntaxInfo { name: "SLASH" },
+            CARET => &SyntaxInfo { name: "CARET" },
+            PERCENT => &SyntaxInfo { name: "PERCENT" },
+            DOT => &SyntaxInfo { name: "DOT" },
+            DOTDOT => &SyntaxInfo { name: "DOTDOT" },
+            DOTDOTDOT => &SyntaxInfo { name: "DOTDOTDOT" },
+            DOTDOTEQ => &SyntaxInfo { name: "DOTDOTEQ" },
            COLON => &SyntaxInfo { name: "COLON" },
            COLONCOLON => &SyntaxInfo { name: "COLONCOLON" },
-            DOLLAR => &SyntaxInfo { name: "DOLLAR" },
            EQ => &SyntaxInfo { name: "EQ" },
            EQEQ => &SyntaxInfo { name: "EQEQ" },
            FAT_ARROW => &SyntaxInfo { name: "FAT_ARROW" },
-            NEQ => &SyntaxInfo { name: "NEQ" },
            EXCL => &SyntaxInfo { name: "EXCL" },
+            NEQ => &SyntaxInfo { name: "NEQ" },
+            MINUS => &SyntaxInfo { name: "MINUS" },
+            THIN_ARROW => &SyntaxInfo { name: "THIN_ARROW" },
+            ERROR => &SyntaxInfo { name: "ERROR" },
+            IDENT => &SyntaxInfo { name: "IDENT" },
+            UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" },
+            WHITESPACE => &SyntaxInfo { name: "WHITESPACE" },
+            INT_NUMBER => &SyntaxInfo { name: "INT_NUMBER" },
+            FLOAT_NUMBER => &SyntaxInfo { name: "FLOAT_NUMBER" },
            LIFETIME => &SyntaxInfo { name: "LIFETIME" },
            CHAR => &SyntaxInfo { name: "CHAR" },
            BYTE => &SyntaxInfo { name: "BYTE" },
@ -190,15 +199,6 @@ impl SyntaxKind {
            RAW_STRING => &SyntaxInfo { name: "RAW_STRING" },
            BYTE_STRING => &SyntaxInfo { name: "BYTE_STRING" },
            RAW_BYTE_STRING => &SyntaxInfo { name: "RAW_BYTE_STRING" },
-            PLUS => &SyntaxInfo { name: "PLUS" },
-            MINUS => &SyntaxInfo { name: "MINUS" },
-            STAR => &SyntaxInfo { name: "STAR" },
-            SLASH => &SyntaxInfo { name: "SLASH" },
-            CARET => &SyntaxInfo { name: "CARET" },
-            PERCENT => &SyntaxInfo { name: "PERCENT" },
-            AMPERSAND => &SyntaxInfo { name: "AMPERSAND" },
-            PIPE => &SyntaxInfo { name: "PIPE" },
-            THIN_ARROW => &SyntaxInfo { name: "THIN_ARROW" },
            COMMENT => &SyntaxInfo { name: "COMMENT" },
            DOC_COMMENT => &SyntaxInfo { name: "DOC_COMMENT" },
            SHEBANG => &SyntaxInfo { name: "SHEBANG" },
@ -325,5 +325,34 @@ impl SyntaxKind {
        };
        Some(kw)
    }
+
+    pub(crate) fn from_char(c: char) -> Option<SyntaxKind> {
+        let tok = match c {
+            ';' => SEMI,
+            ',' => COMMA,
+            '(' => L_PAREN,
+            ')' => R_PAREN,
+            '{' => L_CURLY,
+            '}' => R_CURLY,
+            '[' => L_BRACK,
+            ']' => R_BRACK,
+            '<' => L_ANGLE,
+            '>' => R_ANGLE,
+            '@' => AT,
+            '#' => POUND,
+            '~' => TILDE,
+            '?' => QUESTION,
+            '$' => DOLLAR,
+            '&' => AMPERSAND,
+            '|' => PIPE,
+            '+' => PLUS,
+            '*' => STAR,
+            '/' => SLASH,
+            '^' => CARET,
+            '%' => PERCENT,
+            _ => return None,
+        };
+        Some(tok)
+    }
 }

--- a/src/syntax_kinds/generated.rs.tera
+++ b/src/syntax_kinds/generated.rs.tera
@ -5,7 +5,13 @@ use super::SyntaxInfo;
 /// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum SyntaxKind {
-{%- for t in tokens %}
+{%- for t in single_byte_tokens %}
+    {{t.1}},
+{%- endfor -%}
+{% for t in multi_byte_tokens %}
+    {{t.1}},
+{%- endfor -%}
+{% for t in tokens %}
    {{t}},
 {%- endfor -%}
 {% for kw in keywords %}
@ -29,7 +35,13 @@ use self::SyntaxKind::*;
 impl SyntaxKind {
    pub(crate) fn info(self) -> &'static SyntaxInfo {
        match self {
-{%- for t in tokens %}
+{%- for t in single_byte_tokens %}
+            {{t.1}} => &SyntaxInfo { name: "{{t.1}}" },
+{%- endfor -%}
+{% for t in multi_byte_tokens %}
+            {{t.1}} => &SyntaxInfo { name: "{{t.1}}" },
+{%- endfor -%}
+{% for t in tokens %}
            {{t}} => &SyntaxInfo { name: "{{t}}" },
 {%- endfor -%}
 {% for kw in keywords %}
@ -55,5 +67,15 @@ impl SyntaxKind {
        };
        Some(kw)
    }
+
+    pub(crate) fn from_char(c: char) -> Option<SyntaxKind> {
+        let tok = match c {
+{%- for t in single_byte_tokens %}
+            '{{t.0}}' => {{t.1}},
+{%- endfor %}
+            _ => return None,
+        };
+        Some(tok)
+    }
 }

--- a/tools/src/main.rs
+++ b/tools/src/main.rs
@ -64,7 +64,7 @@ fn get_kinds() -> Result<String> {
    let grammar = grammar()?;
    let template = fs::read_to_string(SYNTAX_KINDS_TEMPLATE)?;
    let ret = tera::Tera::one_off(&template, &grammar, false)
-        .map_err(|e| format_err!("template error: {}", e))?;
+        .map_err(|e| format_err!("template error: {:?}", e))?;
    Ok(ret)
 }