start SOA parser interface

crates/parser/src/lib.rs

@@ -20,6 +20,7 @@ mod syntax_kind;
 mod event;
 mod parser;
 mod grammar;
+mod tokens;
 
 pub(crate) use token_set::TokenSet;
 

crates/parser/src/tokens.rs

@@ -0,0 +1,58 @@
+use crate::SyntaxKind;
+
+type bits = u64;
+
+pub type IdentKind = u8;
+
+/// Main input to the parser.
+///
+/// A sequence of tokens represented internally as a struct of arrays.
+#[derive(Default)]
+pub struct Tokens {
+    kind: Vec<SyntaxKind>,
+    joint: Vec<bits>,
+    ident_kind: Vec<IdentKind>,
+}
+
+impl Tokens {
+    pub fn push(&mut self, was_joint: bool, kind: SyntaxKind) {
+        self.push_impl(was_joint, kind, 0)
+    }
+    pub fn push_ident(&mut self, ident_kind: IdentKind) {
+        self.push_impl(false, SyntaxKind::IDENT, ident_kind)
+    }
+    fn push_impl(&mut self, was_joint: bool, kind: SyntaxKind, ctx: IdentKind) {
+        let idx = self.len();
+        if idx % (bits::BITS as usize) == 0 {
+            self.joint.push(0);
+        }
+        if was_joint && idx > 0 {
+            self.set_joint(idx - 1);
+        }
+        self.kind.push(kind);
+        self.ident_kind.push(ctx);
+    }
+    fn set_joint(&mut self, n: usize) {
+        let (idx, b_idx) = self.bit_index(n);
+        self.joint[idx] |= 1 << b_idx;
+    }
+    fn get_joint(&self, n: usize) -> bool {
+        let (idx, b_idx) = self.bit_index(n);
+        self.joint[idx] & 1 << b_idx != 0
+    }
+    fn bit_index(&self, n: usize) -> (usize, usize) {
+        let idx = n / (bits::BITS as usize);
+        let b_idx = n % (bits::BITS as usize);
+        (idx, b_idx)
+    }
+
+    pub fn len(&self) -> usize {
+        self.kind.len()
+    }
+    pub(crate) fn get(&self, idx: usize) -> Option<(SyntaxKind, bool, IdentKind)> {
+        let kind = *self.kind.get(idx)?;
+        let joint = self.get_joint(idx);
+        let ident_kind = *self.ident_kind.get(idx)?;
+        Some((kind, joint, ident_kind))
+    }
+}