Parser: more groundwork

src/parser/event_parser.rs

@@ -0,0 +1,66 @@
+use {Token, TextUnit, SyntaxKind};
+
+use syntax_kinds::*;
+
+
+pub(crate) enum Event {
+    Start { kind: SyntaxKind },
+    Finish,
+    Token {
+        kind: SyntaxKind,
+        n_raw_tokens: u8,
+    }
+}
+
+pub(crate) fn parse<'t>(text: &'t str, raw_tokens: &'t [Token]) -> Vec<Event> {
+    let mut parser = Parser::new(text, raw_tokens);
+    parse_file(&mut parser);
+    parser.events
+}
+
+struct Parser<'t> {
+    text: &'t str,
+    raw_tokens: &'t [Token],
+    non_ws_tokens: Vec<(usize, TextUnit)>,
+
+    pos: usize,
+    events: Vec<Event>,
+}
+
+impl<'t> Parser<'t> {
+    fn new(text: &'t str, raw_tokens: &'t [Token]) -> Parser<'t> {
+        let mut non_ws_tokens = Vec::new();
+        let mut len = TextUnit::new(0);
+        for (idx, &token) in raw_tokens.iter().enumerate() {
+            match token.kind {
+                WHITESPACE | COMMENT => (),
+                _ => non_ws_tokens.push((idx, len)),
+            }
+            len += token.len;
+        }
+
+        Parser {
+            text,
+            raw_tokens,
+            non_ws_tokens,
+
+            pos: 0,
+            events: Vec::new(),
+        }
+    }
+
+    fn start(&mut self, kind: SyntaxKind) {
+        self.event(Event::Start { kind });
+    }
+    fn finish(&mut self) {
+        self.event(Event::Finish);
+    }
+    fn event(&mut self, event: Event) {
+        self.events.push(event)
+    }
+}
+
+fn parse_file(p: &mut Parser) {
+    p.start(FILE);
+    p.finish();
+}

src/parser/mod.rs

@@ -1,11 +1,24 @@
-use {Token, File, FileBuilder, Sink};
+use {Token, File, FileBuilder, Sink, SyntaxKind};
 
 use syntax_kinds::*;
 
+mod event_parser;
+use self::event_parser::Event;
+
 
 pub fn parse(text: String, tokens: &[Token]) -> File {
+    let events = event_parser::parse(&text, tokens);
+    from_events_to_file(text, events)
+}
+
+fn from_events_to_file(text: String, events: Vec<Event>) -> File {
     let mut builder = FileBuilder::new(text);
-    builder.start_internal(FILE);
+    for event in events {
-    builder.finish_internal();
+        match event {
+            Event::Start { kind } => builder.start_internal(kind),
+            Event::Finish => builder.finish_internal(),
+            Event::Token { .. } => unimplemented!(),
+        }
+    }
     builder.finish()
 }