Lexer: underscore

This commit is contained in:
Aleksey Kladov 2017-12-30 01:01:57 +03:00
parent ff5a6549b4
commit 8103772a10
4 changed files with 24 additions and 0 deletions

View file

@ -22,6 +22,14 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
// They are not identifiers, and are handled further down.
let ident_start = is_ident_start(c) && !string_literal_start(c, ptr.next(), ptr.nnext());
if ident_start {
let is_single_letter = match ptr.next() {
None => true,
Some(c) if !is_ident_continue(c) => true,
_ => false,
};
if is_single_letter {
return if c == '_' { UNDERSCORE } else { IDENT };
}
ptr.bump_while(is_ident_continue);
return IDENT;
}

View file

@ -0,0 +1 @@
foo foo_ _foo _ __ x привет

View file

@ -0,0 +1,14 @@
IDENT 3
WHITESPACE 1
IDENT 4
WHITESPACE 1
IDENT 4
WHITESPACE 1
UNDERSCORE 1
WHITESPACE 1
IDENT 2
WHITESPACE 1
IDENT 1
WHITESPACE 1
IDENT 12
WHITESPACE 1

View file

@ -1,4 +1,5 @@
Fixmes:
* Fix `is_whitespace`, add more test
* Add more thorough tests for idents for XID_Start & XID_Continue