diff --git a/grammar.ron b/grammar.ron index c0564e9cf9..482e00f914 100644 --- a/grammar.ron +++ b/grammar.ron @@ -39,5 +39,14 @@ Grammar( "RAW_STRING", "BYTE_STRING", "RAW_BYTE_STRING", + "PLUS", + "MINUS", + "STAR", + "SLASH", + "CARET", + "PERCENT", + "AMPERSAND", + "PIPE", + "THIN_ARROW", ] ) \ No newline at end of file diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index d66046ca1d..be86416c48 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs @@ -59,6 +59,13 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { '~' => return TILDE, '?' => return QUESTION, '$' => return DOLLAR, + '&' => return AMPERSAND, + '|' => return PIPE, + '+' => return PLUS, + '*' => return STAR, + '/' => return SLASH, + '^' => return CARET, + '%' => return PERCENT, // Multi-byte tokens. '.' => return match (ptr.next(), ptr.nnext()) { @@ -103,6 +110,12 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { } _ => NOT, }, + '-' => return if ptr.next_is('>') { + ptr.bump(); + THIN_ARROW + } else { + MINUS + }, // If the character is an ident start not followed by another single // quote, then this is a lifetime name: diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs index 4a68acb31f..83fabe403a 100644 --- a/src/syntax_kinds.rs +++ b/src/syntax_kinds.rs @@ -40,8 +40,17 @@ pub const STRING: SyntaxKind = SyntaxKind(35); pub const RAW_STRING: SyntaxKind = SyntaxKind(36); pub const BYTE_STRING: SyntaxKind = SyntaxKind(37); pub const RAW_BYTE_STRING: SyntaxKind = SyntaxKind(38); +pub const PLUS: SyntaxKind = SyntaxKind(39); +pub const MINUS: SyntaxKind = SyntaxKind(40); +pub const STAR: SyntaxKind = SyntaxKind(41); +pub const SLASH: SyntaxKind = SyntaxKind(42); +pub const CARET: SyntaxKind = SyntaxKind(43); +pub const PERCENT: SyntaxKind = SyntaxKind(44); +pub const AMPERSAND: SyntaxKind = SyntaxKind(45); +pub const PIPE: SyntaxKind = SyntaxKind(46); +pub const THIN_ARROW: SyntaxKind = SyntaxKind(47); -static INFOS: [SyntaxInfo; 39] = [ +static INFOS: [SyntaxInfo; 48] = [ SyntaxInfo { name: "ERROR" }, SyntaxInfo { name: "IDENT" }, SyntaxInfo { name: "UNDERSCORE" }, @@ -81,6 +90,15 @@ static INFOS: [SyntaxInfo; 39] = [ SyntaxInfo { name: "RAW_STRING" }, SyntaxInfo { name: "BYTE_STRING" }, SyntaxInfo { name: "RAW_BYTE_STRING" }, + SyntaxInfo { name: "PLUS" }, + SyntaxInfo { name: "MINUS" }, + SyntaxInfo { name: "STAR" }, + SyntaxInfo { name: "SLASH" }, + SyntaxInfo { name: "CARET" }, + SyntaxInfo { name: "PERCENT" }, + SyntaxInfo { name: "AMPERSAND" }, + SyntaxInfo { name: "PIPE" }, + SyntaxInfo { name: "THIN_ARROW" }, ]; pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo { diff --git a/tests/data/lexer/0004_numbers.txt b/tests/data/lexer/0004_numbers.txt index be8dee48b7..4b5fd9f71c 100644 --- a/tests/data/lexer/0004_numbers.txt +++ b/tests/data/lexer/0004_numbers.txt @@ -47,13 +47,13 @@ L_PAREN 1 "(" R_PAREN 1 ")" WHITESPACE 1 "\n" INT_NUMBER 2 "0e" -ERROR 1 "+" +PLUS 1 "+" INT_NUMBER 1 "1" WHITESPACE 1 "\n" INT_NUMBER 1 "0" DOT 1 "." IDENT 1 "e" -ERROR 1 "+" +PLUS 1 "+" INT_NUMBER 1 "1" WHITESPACE 1 "\n" FLOAT_NUMBER 6 "0.0E-2" diff --git a/tests/data/lexer/0005_symbols.rs b/tests/data/lexer/0005_symbols.rs index c41e985a12..487569b5ae 100644 --- a/tests/data/lexer/0005_symbols.rs +++ b/tests/data/lexer/0005_symbols.rs @@ -1,5 +1,6 @@ -; , ( ) { } [ ] < > @ # ~ ? $ +; , ( ) { } [ ] < > @ # ~ ? $ & | + * / ^ % . .. ... ..= : :: = => -! != \ No newline at end of file +! != +- -> diff --git a/tests/data/lexer/0005_symbols.txt b/tests/data/lexer/0005_symbols.txt index a1b777a92b..ebb41accd4 100644 --- a/tests/data/lexer/0005_symbols.txt +++ b/tests/data/lexer/0005_symbols.txt @@ -27,6 +27,20 @@ WHITESPACE 1 " " QUESTION 1 "?" WHITESPACE 1 " " DOLLAR 1 "$" +WHITESPACE 1 " " +AMPERSAND 1 "&" +WHITESPACE 1 " " +PIPE 1 "|" +WHITESPACE 1 " " +PLUS 1 "+" +WHITESPACE 1 " " +STAR 1 "*" +WHITESPACE 1 " " +SLASH 1 "/" +WHITESPACE 1 " " +CARET 1 "^" +WHITESPACE 1 " " +PERCENT 1 "%" WHITESPACE 1 "\n" DOT 1 "." WHITESPACE 1 " " @@ -47,3 +61,8 @@ WHITESPACE 1 "\n" NOT 1 "!" WHITESPACE 1 " " NEQ 2 "!=" +WHITESPACE 1 "\n" +MINUS 1 "-" +WHITESPACE 1 " " +THIN_ARROW 2 "->" +WHITESPACE 1 "\n"