Lexer: more symbols

grammar.ron

@@ -39,5 +39,14 @@ Grammar(
         "RAW_STRING",
         "BYTE_STRING",
         "RAW_BYTE_STRING",
+        "PLUS",
+        "MINUS",
+        "STAR",
+        "SLASH",
+        "CARET",
+        "PERCENT",
+        "AMPERSAND",
+        "PIPE",
+        "THIN_ARROW",
     ]
 )

src/lexer/mod.rs

@@ -59,6 +59,13 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
         '~' => return TILDE,
         '?' => return QUESTION,
         '$' => return DOLLAR,
+        '&' => return AMPERSAND,
+        '|' => return PIPE,
+        '+' => return PLUS,
+        '*' => return STAR,
+        '/' => return SLASH,
+        '^' => return CARET,
+        '%' => return PERCENT,
 
         // Multi-byte tokens.
         '.' => return match (ptr.next(), ptr.nnext()) {
@@ -103,6 +110,12 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
             }
             _ => NOT,
         },
+        '-' => return if ptr.next_is('>') {
+            ptr.bump();
+            THIN_ARROW
+        } else {
+            MINUS
+        },
 
         // If the character is an ident start not followed by another single
         // quote, then this is a lifetime name:

src/syntax_kinds.rs

@@ -40,8 +40,17 @@ pub const STRING: SyntaxKind = SyntaxKind(35);
 pub const RAW_STRING: SyntaxKind = SyntaxKind(36);
 pub const BYTE_STRING: SyntaxKind = SyntaxKind(37);
 pub const RAW_BYTE_STRING: SyntaxKind = SyntaxKind(38);
+pub const PLUS: SyntaxKind = SyntaxKind(39);
+pub const MINUS: SyntaxKind = SyntaxKind(40);
+pub const STAR: SyntaxKind = SyntaxKind(41);
+pub const SLASH: SyntaxKind = SyntaxKind(42);
+pub const CARET: SyntaxKind = SyntaxKind(43);
+pub const PERCENT: SyntaxKind = SyntaxKind(44);
+pub const AMPERSAND: SyntaxKind = SyntaxKind(45);
+pub const PIPE: SyntaxKind = SyntaxKind(46);
+pub const THIN_ARROW: SyntaxKind = SyntaxKind(47);
 
-static INFOS: [SyntaxInfo; 39] = [
+static INFOS: [SyntaxInfo; 48] = [
     SyntaxInfo { name: "ERROR" },
     SyntaxInfo { name: "IDENT" },
     SyntaxInfo { name: "UNDERSCORE" },
@@ -81,6 +90,15 @@ static INFOS: [SyntaxInfo; 39] = [
     SyntaxInfo { name: "RAW_STRING" },
     SyntaxInfo { name: "BYTE_STRING" },
     SyntaxInfo { name: "RAW_BYTE_STRING" },
+    SyntaxInfo { name: "PLUS" },
+    SyntaxInfo { name: "MINUS" },
+    SyntaxInfo { name: "STAR" },
+    SyntaxInfo { name: "SLASH" },
+    SyntaxInfo { name: "CARET" },
+    SyntaxInfo { name: "PERCENT" },
+    SyntaxInfo { name: "AMPERSAND" },
+    SyntaxInfo { name: "PIPE" },
+    SyntaxInfo { name: "THIN_ARROW" },
 ];
 
 pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo {

tests/data/lexer/0004_numbers.txt

@@ -47,13 +47,13 @@ L_PAREN 1 "("
 R_PAREN 1 ")"
 WHITESPACE 1 "\n"
 INT_NUMBER 2 "0e"
-ERROR 1 "+"
+PLUS 1 "+"
 INT_NUMBER 1 "1"
 WHITESPACE 1 "\n"
 INT_NUMBER 1 "0"
 DOT 1 "."
 IDENT 1 "e"
-ERROR 1 "+"
+PLUS 1 "+"
 INT_NUMBER 1 "1"
 WHITESPACE 1 "\n"
 FLOAT_NUMBER 6 "0.0E-2"

tests/data/lexer/0005_symbols.rs

@@ -1,5 +1,6 @@
-; , ( ) { } [ ] < > @ # ~ ? $
+; , ( ) { } [ ] < > @ # ~ ? $ & | + * / ^ %
 . .. ... ..=
 : ::
 = =>
-! !=
+! !=
+- ->

tests/data/lexer/0005_symbols.txt

@@ -27,6 +27,20 @@ WHITESPACE 1 " "
 QUESTION 1 "?"
 WHITESPACE 1 " "
 DOLLAR 1 "$"
+WHITESPACE 1 " "
+AMPERSAND 1 "&"
+WHITESPACE 1 " "
+PIPE 1 "|"
+WHITESPACE 1 " "
+PLUS 1 "+"
+WHITESPACE 1 " "
+STAR 1 "*"
+WHITESPACE 1 " "
+SLASH 1 "/"
+WHITESPACE 1 " "
+CARET 1 "^"
+WHITESPACE 1 " "
+PERCENT 1 "%"
 WHITESPACE 1 "\n"
 DOT 1 "."
 WHITESPACE 1 " "
@@ -47,3 +61,8 @@ WHITESPACE 1 "\n"
 NOT 1 "!"
 WHITESPACE 1 " "
 NEQ 2 "!="
+WHITESPACE 1 "\n"
+MINUS 1 "-"
+WHITESPACE 1 " "
+THIN_ARROW 2 "->"
+WHITESPACE 1 "\n"