Lexer: underscore

src/lexer/mod.rs

@@ -22,6 +22,14 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
     // They are not identifiers, and are handled further down.
     let ident_start = is_ident_start(c) && !string_literal_start(c, ptr.next(), ptr.nnext());
     if ident_start {
+        let is_single_letter = match ptr.next() {
+            None => true,
+            Some(c) if !is_ident_continue(c) => true,
+            _ => false,
+        };
+        if is_single_letter {
+            return if c == '_' { UNDERSCORE } else { IDENT };
+        }
         ptr.bump_while(is_ident_continue);
         return IDENT;
     }

tests/data/lexer/0003_ident.rs

@@ -0,0 +1 @@
+foo foo_ _foo _ __ x привет

tests/data/lexer/0003_ident.txt

@@ -0,0 +1,14 @@
+IDENT 3
+WHITESPACE 1
+IDENT 4
+WHITESPACE 1
+IDENT 4
+WHITESPACE 1
+UNDERSCORE 1
+WHITESPACE 1
+IDENT 2
+WHITESPACE 1
+IDENT 1
+WHITESPACE 1
+IDENT 12
+WHITESPACE 1

validation.md

@@ -1,4 +1,5 @@
 Fixmes:
 
 * Fix `is_whitespace`, add more test
+* Add more thorough tests for idents for XID_Start & XID_Continue