2021-08-28 17:36:41 +00:00
|
|
|
//! Serialization-friendly representation of `tt::Subtree`.
|
|
|
|
//!
|
|
|
|
//! It is possible to serialize `Subtree` as is, as a tree, but using
|
|
|
|
//! arbitrary-nested trees in JSON is problematic, as they can cause the JSON
|
|
|
|
//! parser to overflow the stack.
|
|
|
|
//!
|
|
|
|
//! Additionally, such implementation would be pretty verbose, and we do care
|
|
|
|
//! about performance here a bit.
|
|
|
|
//!
|
|
|
|
//! So what this module does is dumping a `tt::Subtree` into a bunch of flat
|
|
|
|
//! array of numbers. See the test in the parent module to get an example
|
|
|
|
//! output.
|
|
|
|
//!
|
|
|
|
//! ```json
|
|
|
|
//! {
|
|
|
|
//! // Array of subtrees, each subtree is represented by 4 numbers:
|
|
|
|
//! // id of delimiter, delimiter kind, index of first child in `token_tree`,
|
|
|
|
//! // index of last child in `token_tree`
|
|
|
|
//! "subtree":[4294967295,0,0,5,2,2,5,5],
|
|
|
|
//! // 2 ints per literal: [token id, index into `text`]
|
|
|
|
//! "literal":[4294967295,1],
|
|
|
|
//! // 3 ints per punct: [token id, char, spacing]
|
|
|
|
//! "punct":[4294967295,64,1],
|
|
|
|
//! // 2 ints per ident: [token id, index into `text`]
|
|
|
|
//! "ident": [0,0,1,1],
|
|
|
|
//! // children of all subtrees, concatenated. Each child is represented as `index << 2 | tag`
|
|
|
|
//! // where tag denotes one of subtree, literal, punct or ident.
|
|
|
|
//! "token_tree":[3,7,1,4],
|
|
|
|
//! // Strings shared by idents and literals
|
|
|
|
//! "text": ["struct","Foo"]
|
|
|
|
//! }
|
|
|
|
//! ```
|
|
|
|
//!
|
|
|
|
//! We probably should replace most of the code here with bincode someday, but,
|
|
|
|
//! as we don't have bincode in Cargo.toml yet, lets stick with serde_json for
|
|
|
|
//! the time being.
|
|
|
|
|
2024-02-01 15:16:38 +00:00
|
|
|
use std::collections::VecDeque;
|
2021-08-28 17:36:41 +00:00
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
use indexmap::IndexSet;
|
2023-12-11 11:16:12 +00:00
|
|
|
use la_arena::RawIdx;
|
2024-02-01 15:16:38 +00:00
|
|
|
use rustc_hash::FxHashMap;
|
2021-08-28 17:36:41 +00:00
|
|
|
use serde::{Deserialize, Serialize};
|
2023-12-11 11:16:12 +00:00
|
|
|
use span::{ErasedFileAstId, FileId, Span, SpanAnchor, SyntaxContextId};
|
|
|
|
use text_size::TextRange;
|
2023-01-31 10:49:49 +00:00
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
use crate::msg::ENCODE_CLOSE_SPAN_VERSION;
|
2023-06-29 10:23:45 +00:00
|
|
|
|
2023-12-11 11:16:12 +00:00
|
|
|
pub type SpanDataIndexMap = IndexSet<Span>;
|
|
|
|
|
|
|
|
pub fn serialize_span_data_index_map(map: &SpanDataIndexMap) -> Vec<u32> {
|
|
|
|
map.iter()
|
|
|
|
.flat_map(|span| {
|
|
|
|
[
|
|
|
|
span.anchor.file_id.index(),
|
|
|
|
span.anchor.ast_id.into_raw().into_u32(),
|
|
|
|
span.range.start().into(),
|
|
|
|
span.range.end().into(),
|
|
|
|
span.ctx.into_u32(),
|
|
|
|
]
|
|
|
|
})
|
|
|
|
.collect()
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn deserialize_span_data_index_map(map: &[u32]) -> SpanDataIndexMap {
|
|
|
|
debug_assert!(map.len() % 5 == 0);
|
|
|
|
map.chunks_exact(5)
|
|
|
|
.map(|span| {
|
|
|
|
let &[file_id, ast_id, start, end, e] = span else { unreachable!() };
|
|
|
|
Span {
|
|
|
|
anchor: SpanAnchor {
|
|
|
|
file_id: FileId::from_raw(file_id),
|
|
|
|
ast_id: ErasedFileAstId::from_raw(RawIdx::from_u32(ast_id)),
|
|
|
|
},
|
|
|
|
range: TextRange::new(start.into(), end.into()),
|
|
|
|
ctx: SyntaxContextId::from_u32(e),
|
|
|
|
}
|
|
|
|
})
|
|
|
|
.collect()
|
|
|
|
}
|
2023-11-28 15:28:51 +00:00
|
|
|
|
|
|
|
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
|
|
|
|
pub struct TokenId(pub u32);
|
|
|
|
|
|
|
|
impl std::fmt::Debug for TokenId {
|
|
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
|
|
self.0.fmt(f)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-08-28 17:36:41 +00:00
|
|
|
#[derive(Serialize, Deserialize, Debug)]
|
|
|
|
pub struct FlatTree {
|
|
|
|
subtree: Vec<u32>,
|
|
|
|
literal: Vec<u32>,
|
|
|
|
punct: Vec<u32>,
|
|
|
|
ident: Vec<u32>,
|
|
|
|
token_tree: Vec<u32>,
|
|
|
|
text: Vec<String>,
|
2023-06-29 10:23:45 +00:00
|
|
|
}
|
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
struct SubtreeRepr {
|
|
|
|
open: TokenId,
|
|
|
|
close: TokenId,
|
2023-01-31 10:49:49 +00:00
|
|
|
kind: tt::DelimiterKind,
|
2021-08-28 17:36:41 +00:00
|
|
|
tt: [u32; 2],
|
|
|
|
}
|
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
struct LiteralRepr {
|
|
|
|
id: TokenId,
|
2021-08-28 17:36:41 +00:00
|
|
|
text: u32,
|
|
|
|
}
|
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
struct PunctRepr {
|
|
|
|
id: TokenId,
|
2021-08-28 17:36:41 +00:00
|
|
|
char: char,
|
|
|
|
spacing: tt::Spacing,
|
|
|
|
}
|
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
struct IdentRepr {
|
|
|
|
id: TokenId,
|
2021-08-28 17:36:41 +00:00
|
|
|
text: u32,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl FlatTree {
|
2023-11-28 15:28:51 +00:00
|
|
|
pub fn new(
|
2023-12-18 12:30:41 +00:00
|
|
|
subtree: &tt::Subtree<Span>,
|
2023-06-29 10:23:45 +00:00
|
|
|
version: u32,
|
2023-12-11 11:16:12 +00:00
|
|
|
span_data_table: &mut SpanDataIndexMap,
|
2023-06-29 10:23:45 +00:00
|
|
|
) -> FlatTree {
|
2021-08-28 17:36:41 +00:00
|
|
|
let mut w = Writer {
|
2024-02-01 15:16:38 +00:00
|
|
|
string_table: FxHashMap::default(),
|
2021-08-28 17:36:41 +00:00
|
|
|
work: VecDeque::new(),
|
2023-11-28 15:28:51 +00:00
|
|
|
span_data_table,
|
2021-08-28 17:36:41 +00:00
|
|
|
|
|
|
|
subtree: Vec::new(),
|
|
|
|
literal: Vec::new(),
|
|
|
|
punct: Vec::new(),
|
|
|
|
ident: Vec::new(),
|
|
|
|
token_tree: Vec::new(),
|
|
|
|
text: Vec::new(),
|
|
|
|
};
|
|
|
|
w.write(subtree);
|
2023-11-28 15:28:51 +00:00
|
|
|
|
|
|
|
FlatTree {
|
2023-04-14 08:34:41 +00:00
|
|
|
subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
|
2023-11-28 15:28:51 +00:00
|
|
|
write_vec(w.subtree, SubtreeRepr::write_with_close_span)
|
2023-04-14 08:34:41 +00:00
|
|
|
} else {
|
2023-11-28 15:28:51 +00:00
|
|
|
write_vec(w.subtree, SubtreeRepr::write)
|
2023-04-14 08:34:41 +00:00
|
|
|
},
|
2023-11-28 15:28:51 +00:00
|
|
|
literal: write_vec(w.literal, LiteralRepr::write),
|
|
|
|
punct: write_vec(w.punct, PunctRepr::write),
|
|
|
|
ident: write_vec(w.ident, IdentRepr::write),
|
2021-08-28 17:36:41 +00:00
|
|
|
token_tree: w.token_tree,
|
|
|
|
text: w.text,
|
2023-11-28 15:28:51 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn new_raw(subtree: &tt::Subtree<TokenId>, version: u32) -> FlatTree {
|
|
|
|
let mut w = Writer {
|
2024-02-01 15:16:38 +00:00
|
|
|
string_table: FxHashMap::default(),
|
2023-11-28 15:28:51 +00:00
|
|
|
work: VecDeque::new(),
|
|
|
|
span_data_table: &mut (),
|
|
|
|
|
|
|
|
subtree: Vec::new(),
|
|
|
|
literal: Vec::new(),
|
|
|
|
punct: Vec::new(),
|
|
|
|
ident: Vec::new(),
|
|
|
|
token_tree: Vec::new(),
|
|
|
|
text: Vec::new(),
|
2021-08-28 17:36:41 +00:00
|
|
|
};
|
2023-11-28 15:28:51 +00:00
|
|
|
w.write(subtree);
|
2021-08-28 17:36:41 +00:00
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
FlatTree {
|
|
|
|
subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
|
|
|
|
write_vec(w.subtree, SubtreeRepr::write_with_close_span)
|
|
|
|
} else {
|
|
|
|
write_vec(w.subtree, SubtreeRepr::write)
|
|
|
|
},
|
|
|
|
literal: write_vec(w.literal, LiteralRepr::write),
|
|
|
|
punct: write_vec(w.punct, PunctRepr::write),
|
|
|
|
ident: write_vec(w.ident, IdentRepr::write),
|
|
|
|
token_tree: w.token_tree,
|
|
|
|
text: w.text,
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
pub fn to_subtree_resolved(
|
2023-06-29 10:23:45 +00:00
|
|
|
self,
|
|
|
|
version: u32,
|
2023-12-11 11:16:12 +00:00
|
|
|
span_data_table: &SpanDataIndexMap,
|
2023-12-18 12:30:41 +00:00
|
|
|
) -> tt::Subtree<Span> {
|
2023-11-28 15:28:51 +00:00
|
|
|
Reader {
|
2023-04-14 08:34:41 +00:00
|
|
|
subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
|
2023-11-28 15:28:51 +00:00
|
|
|
read_vec(self.subtree, SubtreeRepr::read_with_close_span)
|
2023-04-14 08:34:41 +00:00
|
|
|
} else {
|
2023-11-28 15:28:51 +00:00
|
|
|
read_vec(self.subtree, SubtreeRepr::read)
|
2023-04-14 08:34:41 +00:00
|
|
|
},
|
2023-11-28 15:28:51 +00:00
|
|
|
literal: read_vec(self.literal, LiteralRepr::read),
|
|
|
|
punct: read_vec(self.punct, PunctRepr::read),
|
|
|
|
ident: read_vec(self.ident, IdentRepr::read),
|
2021-08-28 17:36:41 +00:00
|
|
|
token_tree: self.token_tree,
|
|
|
|
text: self.text,
|
2023-11-28 15:28:51 +00:00
|
|
|
span_data_table,
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
2023-11-28 15:28:51 +00:00
|
|
|
.read()
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn to_subtree_unresolved(self, version: u32) -> tt::Subtree<TokenId> {
|
|
|
|
Reader {
|
|
|
|
subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
|
|
|
|
read_vec(self.subtree, SubtreeRepr::read_with_close_span)
|
|
|
|
} else {
|
|
|
|
read_vec(self.subtree, SubtreeRepr::read)
|
|
|
|
},
|
|
|
|
literal: read_vec(self.literal, LiteralRepr::read),
|
|
|
|
punct: read_vec(self.punct, PunctRepr::read),
|
|
|
|
ident: read_vec(self.ident, IdentRepr::read),
|
|
|
|
token_tree: self.token_tree,
|
|
|
|
text: self.text,
|
|
|
|
span_data_table: &(),
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
2023-11-28 15:28:51 +00:00
|
|
|
.read()
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
fn read_vec<T, F: Fn([u32; N]) -> T, const N: usize>(xs: Vec<u32>, f: F) -> Vec<T> {
|
|
|
|
let mut chunks = xs.chunks_exact(N);
|
|
|
|
let res = chunks.by_ref().map(|chunk| f(chunk.try_into().unwrap())).collect();
|
|
|
|
assert!(chunks.remainder().is_empty());
|
|
|
|
res
|
|
|
|
}
|
|
|
|
|
|
|
|
fn write_vec<T, F: Fn(T) -> [u32; N], const N: usize>(xs: Vec<T>, f: F) -> Vec<u32> {
|
|
|
|
xs.into_iter().flat_map(f).collect()
|
|
|
|
}
|
|
|
|
|
|
|
|
impl SubtreeRepr {
|
|
|
|
fn write(self) -> [u32; 4] {
|
2021-08-28 17:36:41 +00:00
|
|
|
let kind = match self.kind {
|
2023-01-31 10:49:49 +00:00
|
|
|
tt::DelimiterKind::Invisible => 0,
|
|
|
|
tt::DelimiterKind::Parenthesis => 1,
|
|
|
|
tt::DelimiterKind::Brace => 2,
|
|
|
|
tt::DelimiterKind::Bracket => 3,
|
2021-08-28 17:36:41 +00:00
|
|
|
};
|
2023-11-28 15:28:51 +00:00
|
|
|
[self.open.0, kind, self.tt[0], self.tt[1]]
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
2023-11-28 15:28:51 +00:00
|
|
|
fn read([open, kind, lo, len]: [u32; 4]) -> SubtreeRepr {
|
2021-08-28 17:36:41 +00:00
|
|
|
let kind = match kind {
|
2023-01-31 10:49:49 +00:00
|
|
|
0 => tt::DelimiterKind::Invisible,
|
|
|
|
1 => tt::DelimiterKind::Parenthesis,
|
|
|
|
2 => tt::DelimiterKind::Brace,
|
|
|
|
3 => tt::DelimiterKind::Bracket,
|
2022-12-23 18:42:58 +00:00
|
|
|
other => panic!("bad kind {other}"),
|
2021-08-28 17:36:41 +00:00
|
|
|
};
|
2023-11-28 15:28:51 +00:00
|
|
|
SubtreeRepr { open: TokenId(open), close: TokenId(!0), kind, tt: [lo, len] }
|
2023-04-14 08:34:41 +00:00
|
|
|
}
|
2023-11-28 15:28:51 +00:00
|
|
|
fn write_with_close_span(self) -> [u32; 5] {
|
2023-04-14 08:34:41 +00:00
|
|
|
let kind = match self.kind {
|
|
|
|
tt::DelimiterKind::Invisible => 0,
|
|
|
|
tt::DelimiterKind::Parenthesis => 1,
|
|
|
|
tt::DelimiterKind::Brace => 2,
|
|
|
|
tt::DelimiterKind::Bracket => 3,
|
|
|
|
};
|
2023-11-28 15:28:51 +00:00
|
|
|
[self.open.0, self.close.0, kind, self.tt[0], self.tt[1]]
|
2023-04-14 08:34:41 +00:00
|
|
|
}
|
2023-11-28 15:28:51 +00:00
|
|
|
fn read_with_close_span([open, close, kind, lo, len]: [u32; 5]) -> SubtreeRepr {
|
2023-04-14 08:34:41 +00:00
|
|
|
let kind = match kind {
|
|
|
|
0 => tt::DelimiterKind::Invisible,
|
|
|
|
1 => tt::DelimiterKind::Parenthesis,
|
|
|
|
2 => tt::DelimiterKind::Brace,
|
|
|
|
3 => tt::DelimiterKind::Bracket,
|
|
|
|
other => panic!("bad kind {other}"),
|
|
|
|
};
|
2023-11-28 15:28:51 +00:00
|
|
|
SubtreeRepr { open: TokenId(open), close: TokenId(close), kind, tt: [lo, len] }
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
impl LiteralRepr {
|
|
|
|
fn write(self) -> [u32; 2] {
|
|
|
|
[self.id.0, self.text]
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
2023-11-28 15:28:51 +00:00
|
|
|
fn read([id, text]: [u32; 2]) -> LiteralRepr {
|
|
|
|
LiteralRepr { id: TokenId(id), text }
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
impl PunctRepr {
|
|
|
|
fn write(self) -> [u32; 3] {
|
2021-08-28 17:36:41 +00:00
|
|
|
let spacing = match self.spacing {
|
|
|
|
tt::Spacing::Alone => 0,
|
|
|
|
tt::Spacing::Joint => 1,
|
|
|
|
};
|
2023-11-28 15:28:51 +00:00
|
|
|
[self.id.0, self.char as u32, spacing]
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
2023-11-28 15:28:51 +00:00
|
|
|
fn read([id, char, spacing]: [u32; 3]) -> PunctRepr {
|
2021-08-28 17:36:41 +00:00
|
|
|
let spacing = match spacing {
|
|
|
|
0 => tt::Spacing::Alone,
|
|
|
|
1 => tt::Spacing::Joint,
|
2022-12-23 18:42:58 +00:00
|
|
|
other => panic!("bad spacing {other}"),
|
2021-08-28 17:36:41 +00:00
|
|
|
};
|
2023-11-28 15:28:51 +00:00
|
|
|
PunctRepr { id: TokenId(id), char: char.try_into().unwrap(), spacing }
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
impl IdentRepr {
|
|
|
|
fn write(self) -> [u32; 2] {
|
|
|
|
[self.id.0, self.text]
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
2023-11-28 15:28:51 +00:00
|
|
|
fn read(data: [u32; 2]) -> IdentRepr {
|
|
|
|
IdentRepr { id: TokenId(data[0]), text: data[1] }
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-18 12:30:41 +00:00
|
|
|
trait InternableSpan: Copy {
|
2023-11-28 15:28:51 +00:00
|
|
|
type Table;
|
|
|
|
fn token_id_of(table: &mut Self::Table, s: Self) -> TokenId;
|
|
|
|
fn span_for_token_id(table: &Self::Table, id: TokenId) -> Self;
|
|
|
|
}
|
|
|
|
|
2023-12-18 12:30:41 +00:00
|
|
|
impl InternableSpan for TokenId {
|
2023-11-28 15:28:51 +00:00
|
|
|
type Table = ();
|
|
|
|
fn token_id_of((): &mut Self::Table, token_id: Self) -> TokenId {
|
|
|
|
token_id
|
|
|
|
}
|
|
|
|
|
|
|
|
fn span_for_token_id((): &Self::Table, id: TokenId) -> Self {
|
|
|
|
id
|
|
|
|
}
|
|
|
|
}
|
2023-12-18 12:30:41 +00:00
|
|
|
impl InternableSpan for Span {
|
|
|
|
type Table = IndexSet<Span>;
|
2023-11-28 15:28:51 +00:00
|
|
|
fn token_id_of(table: &mut Self::Table, span: Self) -> TokenId {
|
|
|
|
TokenId(table.insert_full(span).0 as u32)
|
|
|
|
}
|
|
|
|
fn span_for_token_id(table: &Self::Table, id: TokenId) -> Self {
|
|
|
|
*table.get_index(id.0 as usize).unwrap_or_else(|| &table[0])
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-18 12:30:41 +00:00
|
|
|
struct Writer<'a, 'span, S: InternableSpan> {
|
2023-06-29 10:23:45 +00:00
|
|
|
work: VecDeque<(usize, &'a tt::Subtree<S>)>,
|
2024-02-01 15:16:38 +00:00
|
|
|
string_table: FxHashMap<&'a str, u32>,
|
2023-11-28 15:28:51 +00:00
|
|
|
span_data_table: &'span mut S::Table,
|
2021-08-28 17:36:41 +00:00
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
subtree: Vec<SubtreeRepr>,
|
|
|
|
literal: Vec<LiteralRepr>,
|
|
|
|
punct: Vec<PunctRepr>,
|
|
|
|
ident: Vec<IdentRepr>,
|
2021-08-28 17:36:41 +00:00
|
|
|
token_tree: Vec<u32>,
|
|
|
|
text: Vec<String>,
|
|
|
|
}
|
|
|
|
|
2023-12-18 12:30:41 +00:00
|
|
|
impl<'a, 'span, S: InternableSpan> Writer<'a, 'span, S> {
|
2023-06-29 10:23:45 +00:00
|
|
|
fn write(&mut self, root: &'a tt::Subtree<S>) {
|
2021-08-28 17:36:41 +00:00
|
|
|
self.enqueue(root);
|
|
|
|
while let Some((idx, subtree)) = self.work.pop_front() {
|
|
|
|
self.subtree(idx, subtree);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-11-28 15:28:51 +00:00
|
|
|
fn token_id_of(&mut self, span: S) -> TokenId {
|
|
|
|
S::token_id_of(self.span_data_table, span)
|
|
|
|
}
|
|
|
|
|
2023-06-29 10:23:45 +00:00
|
|
|
fn subtree(&mut self, idx: usize, subtree: &'a tt::Subtree<S>) {
|
2021-08-28 17:36:41 +00:00
|
|
|
let mut first_tt = self.token_tree.len();
|
|
|
|
let n_tt = subtree.token_trees.len();
|
|
|
|
self.token_tree.resize(first_tt + n_tt, !0);
|
|
|
|
|
|
|
|
self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32];
|
|
|
|
|
2024-02-03 22:47:13 +00:00
|
|
|
for child in subtree.token_trees.iter() {
|
2021-08-28 17:36:41 +00:00
|
|
|
let idx_tag = match child {
|
|
|
|
tt::TokenTree::Subtree(it) => {
|
|
|
|
let idx = self.enqueue(it);
|
2022-12-30 10:02:45 +00:00
|
|
|
idx << 2
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
|
|
|
tt::TokenTree::Leaf(leaf) => match leaf {
|
|
|
|
tt::Leaf::Literal(lit) => {
|
|
|
|
let idx = self.literal.len() as u32;
|
|
|
|
let text = self.intern(&lit.text);
|
2023-11-28 15:28:51 +00:00
|
|
|
let id = self.token_id_of(lit.span);
|
|
|
|
self.literal.push(LiteralRepr { id, text });
|
2021-08-28 17:36:41 +00:00
|
|
|
idx << 2 | 0b01
|
|
|
|
}
|
|
|
|
tt::Leaf::Punct(punct) => {
|
|
|
|
let idx = self.punct.len() as u32;
|
2023-11-28 15:28:51 +00:00
|
|
|
let id = self.token_id_of(punct.span);
|
|
|
|
self.punct.push(PunctRepr { char: punct.char, spacing: punct.spacing, id });
|
2021-08-28 17:36:41 +00:00
|
|
|
idx << 2 | 0b10
|
|
|
|
}
|
|
|
|
tt::Leaf::Ident(ident) => {
|
|
|
|
let idx = self.ident.len() as u32;
|
|
|
|
let text = self.intern(&ident.text);
|
2023-11-28 15:28:51 +00:00
|
|
|
let id = self.token_id_of(ident.span);
|
|
|
|
self.ident.push(IdentRepr { id, text });
|
2021-08-28 17:36:41 +00:00
|
|
|
idx << 2 | 0b11
|
|
|
|
}
|
|
|
|
},
|
|
|
|
};
|
|
|
|
self.token_tree[first_tt] = idx_tag;
|
|
|
|
first_tt += 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-06-29 10:23:45 +00:00
|
|
|
fn enqueue(&mut self, subtree: &'a tt::Subtree<S>) -> u32 {
|
2021-08-28 17:36:41 +00:00
|
|
|
let idx = self.subtree.len();
|
2023-11-28 15:28:51 +00:00
|
|
|
let open = self.token_id_of(subtree.delimiter.open);
|
|
|
|
let close = self.token_id_of(subtree.delimiter.close);
|
2023-01-31 10:49:49 +00:00
|
|
|
let delimiter_kind = subtree.delimiter.kind;
|
2023-04-14 08:34:41 +00:00
|
|
|
self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] });
|
2021-08-28 17:36:41 +00:00
|
|
|
self.work.push_back((idx, subtree));
|
|
|
|
idx as u32
|
|
|
|
}
|
|
|
|
|
|
|
|
pub(crate) fn intern(&mut self, text: &'a str) -> u32 {
|
|
|
|
let table = &mut self.text;
|
|
|
|
*self.string_table.entry(text).or_insert_with(|| {
|
|
|
|
let idx = table.len();
|
2024-02-09 15:38:39 +00:00
|
|
|
table.push(text.to_owned());
|
2021-08-28 17:36:41 +00:00
|
|
|
idx as u32
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-18 12:30:41 +00:00
|
|
|
struct Reader<'span, S: InternableSpan> {
|
2023-11-28 15:28:51 +00:00
|
|
|
subtree: Vec<SubtreeRepr>,
|
|
|
|
literal: Vec<LiteralRepr>,
|
|
|
|
punct: Vec<PunctRepr>,
|
|
|
|
ident: Vec<IdentRepr>,
|
2021-08-28 17:36:41 +00:00
|
|
|
token_tree: Vec<u32>,
|
|
|
|
text: Vec<String>,
|
2023-11-28 15:28:51 +00:00
|
|
|
span_data_table: &'span S::Table,
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
|
|
|
|
2023-12-18 12:30:41 +00:00
|
|
|
impl<'span, S: InternableSpan> Reader<'span, S> {
|
2023-06-29 10:23:45 +00:00
|
|
|
pub(crate) fn read(self) -> tt::Subtree<S> {
|
|
|
|
let mut res: Vec<Option<tt::Subtree<S>>> = vec![None; self.subtree.len()];
|
2023-11-28 15:28:51 +00:00
|
|
|
let read_span = |id| S::span_for_token_id(self.span_data_table, id);
|
2021-08-28 17:36:41 +00:00
|
|
|
for i in (0..self.subtree.len()).rev() {
|
|
|
|
let repr = &self.subtree[i];
|
|
|
|
let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize];
|
|
|
|
let s = tt::Subtree {
|
2023-11-28 15:28:51 +00:00
|
|
|
delimiter: tt::Delimiter {
|
|
|
|
open: read_span(repr.open),
|
|
|
|
close: read_span(repr.close),
|
|
|
|
kind: repr.kind,
|
|
|
|
},
|
2021-08-28 17:36:41 +00:00
|
|
|
token_trees: token_trees
|
|
|
|
.iter()
|
|
|
|
.copied()
|
|
|
|
.map(|idx_tag| {
|
|
|
|
let tag = idx_tag & 0b11;
|
|
|
|
let idx = (idx_tag >> 2) as usize;
|
|
|
|
match tag {
|
|
|
|
// XXX: we iterate subtrees in reverse to guarantee
|
|
|
|
// that this unwrap doesn't fire.
|
|
|
|
0b00 => res[idx].take().unwrap().into(),
|
|
|
|
0b01 => {
|
|
|
|
let repr = &self.literal[idx];
|
|
|
|
tt::Leaf::Literal(tt::Literal {
|
|
|
|
text: self.text[repr.text as usize].as_str().into(),
|
2023-11-28 15:28:51 +00:00
|
|
|
span: read_span(repr.id),
|
2021-08-28 17:36:41 +00:00
|
|
|
})
|
|
|
|
.into()
|
|
|
|
}
|
|
|
|
0b10 => {
|
|
|
|
let repr = &self.punct[idx];
|
|
|
|
tt::Leaf::Punct(tt::Punct {
|
|
|
|
char: repr.char,
|
|
|
|
spacing: repr.spacing,
|
2023-11-28 15:28:51 +00:00
|
|
|
span: read_span(repr.id),
|
2021-08-28 17:36:41 +00:00
|
|
|
})
|
|
|
|
.into()
|
|
|
|
}
|
|
|
|
0b11 => {
|
|
|
|
let repr = &self.ident[idx];
|
|
|
|
tt::Leaf::Ident(tt::Ident {
|
|
|
|
text: self.text[repr.text as usize].as_str().into(),
|
2023-11-28 15:28:51 +00:00
|
|
|
span: read_span(repr.id),
|
2021-08-28 17:36:41 +00:00
|
|
|
})
|
|
|
|
.into()
|
|
|
|
}
|
2022-12-23 18:42:58 +00:00
|
|
|
other => panic!("bad tag: {other}"),
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
|
|
|
})
|
|
|
|
.collect(),
|
|
|
|
};
|
2021-10-03 12:39:43 +00:00
|
|
|
res[i] = Some(s);
|
2021-08-28 17:36:41 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
res[0].take().unwrap()
|
|
|
|
}
|
|
|
|
}
|