Merge #18

18: Comments r=matklad a=matklad

src/bin/gen.rs

@@ -51,8 +51,12 @@ impl Grammar {
             write!(acc, "    {},\n", scream(kind)).unwrap();
         }
         acc.push_str("\n");
-        acc.push_str("    TOMBSTONE = !0 - 1,\n");
+        acc.push_str("    // Technical SyntaxKinds: they appear temporally during parsing,\n");
-        acc.push_str("    EOF = !0,\n");
+        acc.push_str("    // but never end up in the final tree\n");
+        acc.push_str("    #[doc(hidden)]\n");
+        acc.push_str("    TOMBSTONE,\n");
+        acc.push_str("    #[doc(hidden)]\n");
+        acc.push_str("    EOF,\n");
         acc.push_str("}\n");
         acc.push_str("pub(crate) use self::SyntaxKind::*;\n");
         acc.push_str("\n");

src/parser/event_parser/mod.rs

@@ -4,17 +4,64 @@ use {SyntaxKind, Token};
 mod parser;
 mod grammar;
 
+/// `Parser` produces a flat list of `Event`s.
+/// They are converted to a tree-structure in
+/// a separate pass, via `TreeBuilder`.
 #[derive(Debug)]
 pub(crate) enum Event {
+    /// This event signifies the start of the node.
+    /// It should be either abandoned (in which case the
+    /// `kind` is `TOMBSTONE`, and the event is ignored),
+    /// or completed via a `Finish` event.
+    ///
+    /// All tokens between a `Start` and a `Finish` would
+    /// become the children of the respective node.
+    ///
+    /// For left-recursive syntactic constructs, the parser produces
+    /// a child node before it sees a parent. `forward_parent`
+    /// exists to allow to tweak parent-child relationships.
+    ///
+    /// Consider this path
+    ///
+    /// foo::bar
+    ///
+    /// The events for it would look like this:
+    ///
+    ///
+    /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH
+    ///       |                          /\
+    ///       |                          |
+    ///       +------forward-parent------+
+    ///
+    /// And the tree would look like this
+    ///
+    ///    +--PATH---------+
+    ///    |   |           |
+    ///    |   |           |
+    ///    |  '::'       'bar'
+    ///    |
+    ///   PATH
+    ///    |
+    ///   'foo'
+    ///
+    /// See also `CompleteMarker::precede`.
     Start {
         kind: SyntaxKind,
         forward_parent: Option<u32>,
     },
+
+    /// Complete the previous `Start` event
     Finish,
+
+    /// Produce a single leaf-element.
+    /// `n_raw_tokens` is used to glue complex contextual tokens.
+    /// For example, lexer tokenizes `>>` as `>`, `>`, and
+    /// `n_raw_tokens = 2` is used to produced a single `>>`.
     Token {
         kind: SyntaxKind,
         n_raw_tokens: u8,
     },
+
     Error {
         message: String,
     },

src/syntax_kinds.rs

@@ -92,8 +92,12 @@ pub enum SyntaxKind {
     ALIAS,
     VISIBILITY,
 
-    TOMBSTONE = !0 - 1,
+    // Technical SyntaxKinds: they appear temporally during parsing,
-    EOF = !0,
+    // but never end up in the final tree
+    #[doc(hidden)]
+    TOMBSTONE,
+    #[doc(hidden)]
+    EOF,
 }
 pub(crate) use self::SyntaxKind::*;