More readable lexer tests

This commit is contained in:
Aleksey Kladov 2017-12-30 15:29:09 +03:00
parent ddc637c161
commit 2d5217dbb7
7 changed files with 103 additions and 98 deletions

View file

@ -56,6 +56,7 @@ impl<'s> Ptr<'s> {
}
fn chars(&self) -> Chars {
self.text[self.len.0 as usize ..].chars()
let len: u32 = self.len.into();
self.text[len as usize ..].chars()
}
}

View file

@ -2,9 +2,7 @@ use std::fmt;
use std::ops;
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct TextUnit(
pub(crate) u32
);
pub struct TextUnit(u32);
impl TextUnit {
pub fn len_of_char(c: char) -> TextUnit {

View file

@ -1,3 +1,3 @@
IDENT 5
WHITESPACE 1
IDENT 5
IDENT 5 "hello"
WHITESPACE 1 " "
IDENT 5 "world"

View file

@ -1,12 +1,12 @@
IDENT 1
WHITESPACE 1
IDENT 1
WHITESPACE 2
IDENT 1
WHITESPACE 1
IDENT 1
WHITESPACE 2
IDENT 1
WHITESPACE 1
IDENT 1
WHITESPACE 1
IDENT 1 "a"
WHITESPACE 1 " "
IDENT 1 "b"
WHITESPACE 2 " "
IDENT 1 "c"
WHITESPACE 1 "\n"
IDENT 1 "d"
WHITESPACE 2 "\n\n"
IDENT 1 "e"
WHITESPACE 1 "\t"
IDENT 1 "f"
WHITESPACE 1 "\n"

View file

@ -1,14 +1,14 @@
IDENT 3
WHITESPACE 1
IDENT 4
WHITESPACE 1
IDENT 4
WHITESPACE 1
UNDERSCORE 1
WHITESPACE 1
IDENT 2
WHITESPACE 1
IDENT 1
WHITESPACE 1
IDENT 12
WHITESPACE 1
IDENT 3 "foo"
WHITESPACE 1 " "
IDENT 4 "foo_"
WHITESPACE 1 " "
IDENT 4 "_foo"
WHITESPACE 1 " "
UNDERSCORE 1 "_"
WHITESPACE 1 " "
IDENT 2 "__"
WHITESPACE 1 " "
IDENT 1 "x"
WHITESPACE 1 " "
IDENT 12 "привет"
WHITESPACE 1 "\n"

View file

@ -1,62 +1,62 @@
INT_NUMBER 1
WHITESPACE 1
INT_NUMBER 2
WHITESPACE 1
INT_NUMBER 2
WHITESPACE 1
INT_NUMBER 2
WHITESPACE 1
INT_NUMBER 2
WHITESPACE 1
INT_NUMBER 2
WHITESPACE 1
FLOAT_NUMBER 2
WHITESPACE 1
INT_NUMBER 2
WHITESPACE 1
INT_NUMBER 2
WHITESPACE 1
INT_NUMBER 1
IDENT 1
WHITESPACE 1
INT_NUMBER 5
WHITESPACE 1
INT_NUMBER 6
WHITESPACE 1
INT_NUMBER 6
WHITESPACE 1
INT_NUMBER 18
WHITESPACE 1
INT_NUMBER 6
WHITESPACE 1
INT_NUMBER 6
WHITESPACE 1
FLOAT_NUMBER 6
WHITESPACE 1
INT_NUMBER 6
WHITESPACE 1
INT_NUMBER 6
WHITESPACE 1
INT_NUMBER 1
ERROR 1
ERROR 1
INT_NUMBER 1
WHITESPACE 1
INT_NUMBER 1
ERROR 1
IDENT 3
ERROR 1
ERROR 1
WHITESPACE 1
INT_NUMBER 2
ERROR 1
INT_NUMBER 1
WHITESPACE 1
INT_NUMBER 1
ERROR 1
IDENT 1
ERROR 1
INT_NUMBER 1
WHITESPACE 1
FLOAT_NUMBER 6
WHITESPACE 1
INT_NUMBER 1 "0"
WHITESPACE 1 " "
INT_NUMBER 2 "0b"
WHITESPACE 1 " "
INT_NUMBER 2 "0o"
WHITESPACE 1 " "
INT_NUMBER 2 "0x"
WHITESPACE 1 " "
INT_NUMBER 2 "00"
WHITESPACE 1 " "
INT_NUMBER 2 "0_"
WHITESPACE 1 " "
FLOAT_NUMBER 2 "0."
WHITESPACE 1 " "
INT_NUMBER 2 "0e"
WHITESPACE 1 " "
INT_NUMBER 2 "0E"
WHITESPACE 1 " "
INT_NUMBER 1 "0"
IDENT 1 "z"
WHITESPACE 1 "\n"
INT_NUMBER 5 "01790"
WHITESPACE 1 " "
INT_NUMBER 6 "0b1790"
WHITESPACE 1 " "
INT_NUMBER 6 "0o1790"
WHITESPACE 1 " "
INT_NUMBER 18 "0x1790aAbBcCdDeEfF"
WHITESPACE 1 " "
INT_NUMBER 6 "001279"
WHITESPACE 1 " "
INT_NUMBER 6 "0_1279"
WHITESPACE 1 " "
FLOAT_NUMBER 6 "0.1279"
WHITESPACE 1 " "
INT_NUMBER 6 "0e1279"
WHITESPACE 1 " "
INT_NUMBER 6 "0E1279"
WHITESPACE 1 "\n"
INT_NUMBER 1 "0"
ERROR 1 "."
ERROR 1 "."
INT_NUMBER 1 "2"
WHITESPACE 1 "\n"
INT_NUMBER 1 "0"
ERROR 1 "."
IDENT 3 "foo"
ERROR 1 "("
ERROR 1 ")"
WHITESPACE 1 "\n"
INT_NUMBER 2 "0e"
ERROR 1 "+"
INT_NUMBER 1 "1"
WHITESPACE 1 "\n"
INT_NUMBER 1 "0"
ERROR 1 "."
IDENT 1 "e"
ERROR 1 "+"
INT_NUMBER 1 "1"
WHITESPACE 1 "\n"
FLOAT_NUMBER 6 "0.0E-2"
WHITESPACE 1 "\n"

View file

@ -31,6 +31,7 @@ fn lexer_test_cases() -> Vec<PathBuf> {
acc.push(path);
}
}
acc.sort();
acc
}
@ -38,7 +39,7 @@ fn lexer_test_case(path: &Path) {
let actual = {
let text = file::get_text(path).unwrap();
let tokens = tokenize(&text);
dump_tokens(&tokens)
dump_tokens(&tokens, &text)
};
let expected = file::get_text(&path.with_extension("txt")).unwrap();
let expected = expected.as_str();
@ -64,10 +65,15 @@ fn tokenize(text: &str) -> Vec<Token> {
acc
}
fn dump_tokens(tokens: &[Token]) -> String {
fn dump_tokens(tokens: &[Token], text: &str) -> String {
let mut acc = String::new();
let mut offset = 0;
for token in tokens {
write!(acc, "{:?} {}\n", token.kind, token.len).unwrap()
let len: u32 = token.len.into();
let len = len as usize;
let token_text = &text[offset..offset + len];
offset += len;
write!(acc, "{:?} {} {:?}\n", token.kind, token.len, token_text).unwrap()
}
acc
}