diff --git a/src/bin/gen.rs b/src/bin/gen.rs index 4b8a5afecf..89609bd77e 100644 --- a/src/bin/gen.rs +++ b/src/bin/gen.rs @@ -51,8 +51,12 @@ impl Grammar { write!(acc, " {},\n", scream(kind)).unwrap(); } acc.push_str("\n"); - acc.push_str(" TOMBSTONE = !0 - 1,\n"); - acc.push_str(" EOF = !0,\n"); + acc.push_str(" // Technical SyntaxKinds: they appear temporally during parsing,\n"); + acc.push_str(" // but never end up in the final tree\n"); + acc.push_str(" #[doc(hidden)]\n"); + acc.push_str(" TOMBSTONE,\n"); + acc.push_str(" #[doc(hidden)]\n"); + acc.push_str(" EOF,\n"); acc.push_str("}\n"); acc.push_str("pub(crate) use self::SyntaxKind::*;\n"); acc.push_str("\n"); diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs index 65aea017b1..7823c476c2 100644 --- a/src/parser/event_parser/mod.rs +++ b/src/parser/event_parser/mod.rs @@ -4,17 +4,64 @@ use {SyntaxKind, Token}; mod parser; mod grammar; +/// `Parser` produces a flat list of `Event`s. +/// They are converted to a tree-structure in +/// a separate pass, via `TreeBuilder`. #[derive(Debug)] pub(crate) enum Event { + /// This event signifies the start of the node. + /// It should be either abandoned (in which case the + /// `kind` is `TOMBSTONE`, and the event is ignored), + /// or completed via a `Finish` event. + /// + /// All tokens between a `Start` and a `Finish` would + /// become the children of the respective node. + /// + /// For left-recursive syntactic constructs, the parser produces + /// a child node before it sees a parent. `forward_parent` + /// exists to allow to tweak parent-child relationships. + /// + /// Consider this path + /// + /// foo::bar + /// + /// The events for it would look like this: + /// + /// + /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH + /// | /\ + /// | | + /// +------forward-parent------+ + /// + /// And the tree would look like this + /// + /// +--PATH---------+ + /// | | | + /// | | | + /// | '::' 'bar' + /// | + /// PATH + /// | + /// 'foo' + /// + /// See also `CompleteMarker::precede`. Start { kind: SyntaxKind, forward_parent: Option, }, + + /// Complete the previous `Start` event Finish, + + /// Produce a single leaf-element. + /// `n_raw_tokens` is used to glue complex contextual tokens. + /// For example, lexer tokenizes `>>` as `>`, `>`, and + /// `n_raw_tokens = 2` is used to produced a single `>>`. Token { kind: SyntaxKind, n_raw_tokens: u8, }, + Error { message: String, }, diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs index aa19c2adff..cc9e74f8e8 100644 --- a/src/syntax_kinds.rs +++ b/src/syntax_kinds.rs @@ -92,8 +92,12 @@ pub enum SyntaxKind { ALIAS, VISIBILITY, - TOMBSTONE = !0 - 1, - EOF = !0, + // Technical SyntaxKinds: they appear temporally during parsing, + // but never end up in the final tree + #[doc(hidden)] + TOMBSTONE, + #[doc(hidden)] + EOF, } pub(crate) use self::SyntaxKind::*;