This commit is contained in:
Aleksey Kladov 2018-07-30 14:08:06 +03:00
parent 6983091d6d
commit 1edb58a802
20 changed files with 130 additions and 141 deletions

View file

@ -6,7 +6,7 @@ matrix:
before_script: before_script:
- rustup component add rustfmt-preview - rustup component add rustfmt-preview
script: script:
- cargo fmt --all -- --write-mode=diff - cargo fmt --all -- --check
- cargo test - cargo test
- cargo gen-kinds --verify - cargo gen-kinds --verify
- cargo gen-tests --verify - cargo gen-tests --verify

View file

@ -1,12 +1,17 @@
use unicode_xid::UnicodeXID; use unicode_xid::UnicodeXID;
pub fn is_ident_start(c: char) -> bool { pub fn is_ident_start(c: char) -> bool {
(c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' (c >= 'a' && c <= 'z')
|| (c >= 'A' && c <= 'Z')
|| c == '_'
|| (c > '\x7f' && UnicodeXID::is_xid_start(c)) || (c > '\x7f' && UnicodeXID::is_xid_start(c))
} }
pub fn is_ident_continue(c: char) -> bool { pub fn is_ident_continue(c: char) -> bool {
(c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' (c >= 'a' && c <= 'z')
|| (c >= 'A' && c <= 'Z')
|| (c >= '0' && c <= '9')
|| c == '_'
|| (c > '\x7f' && UnicodeXID::is_xid_continue(c)) || (c > '\x7f' && UnicodeXID::is_xid_continue(c))
} }

View file

@ -1,22 +1,22 @@
mod ptr;
mod comments;
mod strings;
mod numbers;
mod classes; mod classes;
mod comments;
mod numbers;
mod ptr;
mod strings;
use { use {
TextUnit,
SyntaxKind::{self, *}, SyntaxKind::{self, *},
TextUnit,
}; };
use self::{ use self::{
ptr::Ptr,
classes::*, classes::*,
numbers::scan_number,
strings::{
is_string_literal_start, scan_byte_char_or_string, scan_char,
scan_raw_string, scan_string},
comments::{scan_comment, scan_shebang}, comments::{scan_comment, scan_shebang},
numbers::scan_number,
ptr::Ptr,
strings::{
is_string_literal_start, scan_byte_char_or_string, scan_char, scan_raw_string, scan_string,
},
}; };
/// A token of Rust source. /// A token of Rust source.

View file

@ -1,5 +1,5 @@
use lexer::ptr::Ptr;
use lexer::classes::*; use lexer::classes::*;
use lexer::ptr::Ptr;
use SyntaxKind::{self, *}; use SyntaxKind::{self, *};

View file

@ -11,44 +11,42 @@
//! [rfc#2256]: <https://github.com/rust-lang/rfcs/pull/2256> //! [rfc#2256]: <https://github.com/rust-lang/rfcs/pull/2256>
//! [RFC.md]: <https://github.com/matklad/libsyntax2/blob/master/docs/RFC.md> //! [RFC.md]: <https://github.com/matklad/libsyntax2/blob/master/docs/RFC.md>
#![forbid(missing_debug_implementations, unconditional_recursion, future_incompatible)] #![forbid(
missing_debug_implementations,
unconditional_recursion,
future_incompatible
)]
#![deny(bad_style, missing_docs)] #![deny(bad_style, missing_docs)]
#![allow(missing_docs)] #![allow(missing_docs)]
//#![warn(unreachable_pub)] // rust-lang/rust#47816 //#![warn(unreachable_pub)] // rust-lang/rust#47816
extern crate unicode_xid;
extern crate text_unit; extern crate text_unit;
extern crate unicode_xid;
mod lexer; mod lexer;
mod parser; mod parser;
mod yellow;
mod syntax_kinds; mod syntax_kinds;
mod yellow;
pub use { pub use {
text_unit::{TextRange, TextUnit},
syntax_kinds::SyntaxKind,
yellow::{SyntaxNode, SyntaxNodeRef},
lexer::{tokenize, Token}, lexer::{tokenize, Token},
syntax_kinds::SyntaxKind,
text_unit::{TextRange, TextUnit},
yellow::{SyntaxNode, SyntaxNodeRef},
}; };
pub(crate) use { pub(crate) use yellow::SyntaxError;
yellow::SyntaxError
};
pub fn parse(text: String) -> SyntaxNode { pub fn parse(text: String) -> SyntaxNode {
let tokens = tokenize(&text); let tokens = tokenize(&text);
parser::parse::<yellow::GreenBuilder>(text, &tokens) parser::parse::<yellow::GreenBuilder>(text, &tokens)
} }
/// Utilities for simple uses of the parser. /// Utilities for simple uses of the parser.
pub mod utils { pub mod utils {
use std::{ use std::{collections::BTreeSet, fmt::Write};
fmt::Write,
collections::BTreeSet
};
use {SyntaxNode, SyntaxNodeRef, SyntaxError}; use {SyntaxError, SyntaxNode, SyntaxNodeRef};
/// Parse a file and create a string representation of the resulting parse tree. /// Parse a file and create a string representation of the resulting parse tree.
pub fn dump_tree_green(syntax: &SyntaxNode) -> String { pub fn dump_tree_green(syntax: &SyntaxNode) -> String {
@ -58,11 +56,19 @@ pub mod utils {
go(syntax, &mut result, 0, &mut errors); go(syntax, &mut result, 0, &mut errors);
return result; return result;
fn go(node: SyntaxNodeRef, buff: &mut String, level: usize, errors: &mut BTreeSet<SyntaxError>) { fn go(
node: SyntaxNodeRef,
buff: &mut String,
level: usize,
errors: &mut BTreeSet<SyntaxError>,
) {
buff.push_str(&String::from(" ").repeat(level)); buff.push_str(&String::from(" ").repeat(level));
write!(buff, "{:?}\n", node).unwrap(); write!(buff, "{:?}\n", node).unwrap();
let my_errors: Vec<_> = errors.iter().filter(|e| e.offset == node.range().start()) let my_errors: Vec<_> = errors
.cloned().collect(); .iter()
.filter(|e| e.offset == node.range().start())
.cloned()
.collect();
for err in my_errors { for err in my_errors {
errors.remove(&err); errors.remove(&err);
buff.push_str(&String::from(" ").repeat(level)); buff.push_str(&String::from(" ").repeat(level));
@ -73,8 +79,11 @@ pub mod utils {
go(child, buff, level + 1, errors) go(child, buff, level + 1, errors)
} }
let my_errors: Vec<_> = errors.iter().filter(|e| e.offset == node.range().end()) let my_errors: Vec<_> = errors
.cloned().collect(); .iter()
.filter(|e| e.offset == node.range().end())
.cloned()
.collect();
for err in my_errors { for err in my_errors {
errors.remove(&err); errors.remove(&err);
buff.push_str(&String::from(" ").repeat(level)); buff.push_str(&String::from(" ").repeat(level));

View file

@ -8,9 +8,9 @@
//! `start node`, `finish node`, and `FileBuilder` converts //! `start node`, `finish node`, and `FileBuilder` converts
//! this stream to a real tree. //! this stream to a real tree.
use { use {
TextUnit,
SyntaxKind::{self, TOMBSTONE},
lexer::Token, lexer::Token,
SyntaxKind::{self, TOMBSTONE},
TextUnit,
}; };
pub(crate) trait Sink { pub(crate) trait Sink {

View file

@ -1,9 +1,9 @@
use super::*; use super::*;
mod structs;
mod use_item;
mod consts; mod consts;
mod structs;
mod traits; mod traits;
mod use_item;
pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) { pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) {
attributes::inner_attributes(p); attributes::inner_attributes(p);
@ -12,9 +12,8 @@ pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) {
} }
} }
pub(super) const ITEM_FIRST: TokenSet = token_set![ pub(super) const ITEM_FIRST: TokenSet =
EXTERN_KW, MOD_KW, USE_KW, STRUCT_KW, ENUM_KW, FN_KW, PUB_KW, POUND token_set![EXTERN_KW, MOD_KW, USE_KW, STRUCT_KW, ENUM_KW, FN_KW, PUB_KW, POUND];
];
fn item(p: &mut Parser) { fn item(p: &mut Parser) {
let item = p.start(); let item = p.start();

View file

@ -21,20 +21,17 @@
//! After adding a new inline-test, run `cargo collect-tests` to extract //! After adding a new inline-test, run `cargo collect-tests` to extract
//! it as a standalone text-fixture into `tests/data/parser/inline`, and //! it as a standalone text-fixture into `tests/data/parser/inline`, and
//! run `cargo test` once to create the "gold" value. //! run `cargo test` once to create the "gold" value.
mod items;
mod attributes; mod attributes;
mod expressions; mod expressions;
mod types; mod items;
mod patterns;
mod paths; mod paths;
mod patterns;
mod type_params; mod type_params;
mod types;
use { use {
parser::{parser::Parser, token_set::TokenSet},
SyntaxKind::{self, *}, SyntaxKind::{self, *},
parser::{
parser::Parser,
token_set::TokenSet
}
}; };
pub(crate) fn file(p: &mut Parser) { pub(crate) fn file(p: &mut Parser) {

View file

@ -1,8 +1,4 @@
use { use {lexer::Token, SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit};
SyntaxKind, TextRange, TextUnit,
SyntaxKind::EOF,
lexer::Token,
};
use std::ops::{Add, AddAssign}; use std::ops::{Add, AddAssign};

View file

@ -1,18 +1,14 @@
#[macro_use] #[macro_use]
mod token_set; mod token_set;
mod parser;
mod input;
mod event; mod event;
mod grammar; mod grammar;
mod input;
mod parser;
use { use {lexer::Token, parser::event::process};
lexer::Token,
parser::event::{process}
};
pub(crate) use self::event::Sink; pub(crate) use self::event::Sink;
/// Parse a sequence of tokens into the representative node tree /// Parse a sequence of tokens into the representative node tree
pub(crate) fn parse<S: Sink>(text: String, tokens: &[Token]) -> S::Tree { pub(crate) fn parse<S: Sink>(text: String, tokens: &[Token]) -> S::Tree {
let events = { let events = {

View file

@ -1,5 +1,5 @@
use parser::input::{InputPosition, ParserInput};
use parser::event::Event; use parser::event::Event;
use parser::input::{InputPosition, ParserInput};
use SyntaxKind::{self, EOF, TOMBSTONE}; use SyntaxKind::{self, EOF, TOMBSTONE};

View file

@ -1,7 +1,7 @@
mod generated; mod generated;
use std::fmt; use std::fmt;
use ::{SyntaxKind::*}; use SyntaxKind::*;
pub use self::generated::SyntaxKind; pub use self::generated::SyntaxKind;
@ -16,7 +16,6 @@ pub(crate) struct SyntaxInfo {
pub name: &'static str, pub name: &'static str,
} }
impl SyntaxKind { impl SyntaxKind {
pub(crate) fn is_trivia(self: SyntaxKind) -> bool { pub(crate) fn is_trivia(self: SyntaxKind) -> bool {
match self { match self {

View file

@ -1,7 +1,7 @@
use { use {
parser::Sink,
yellow::{GreenNode, GreenNodeBuilder, SyntaxError, SyntaxNode, SyntaxRoot},
SyntaxKind, TextRange, TextUnit, SyntaxKind, TextRange, TextUnit,
yellow::{SyntaxNode, SyntaxRoot, GreenNode, GreenNodeBuilder, SyntaxError},
parser::Sink
}; };
pub(crate) struct GreenBuilder { pub(crate) struct GreenBuilder {
@ -12,9 +12,7 @@ pub(crate) struct GreenBuilder {
errors: Vec<SyntaxError>, errors: Vec<SyntaxError>,
} }
impl GreenBuilder { impl GreenBuilder {}
}
impl Sink for GreenBuilder { impl Sink for GreenBuilder {
type Tree = SyntaxNode; type Tree = SyntaxNode;
@ -53,7 +51,10 @@ impl Sink for GreenBuilder {
} }
fn error(&mut self, message: String) { fn error(&mut self, message: String) {
self.errors.push(SyntaxError { message, offset: self.pos }) self.errors.push(SyntaxError {
message,
offset: self.pos,
})
} }
fn finish(self) -> SyntaxNode { fn finish(self) -> SyntaxNode {
@ -61,5 +62,3 @@ impl Sink for GreenBuilder {
SyntaxNode::new_owned(root) SyntaxNode::new_owned(root)
} }
} }

View file

@ -1,5 +1,8 @@
use std::sync::Arc; use std::sync::Arc;
use {SyntaxKind::{self, *}, TextUnit}; use {
SyntaxKind::{self, *},
TextUnit,
};
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub(crate) enum GreenNode { pub(crate) enum GreenNode {
@ -36,9 +39,7 @@ impl GreenNode {
fn go(node: &GreenNode, buff: &mut String) { fn go(node: &GreenNode, buff: &mut String) {
match node { match node {
GreenNode::Leaf(l) => buff.push_str(&l.text()), GreenNode::Leaf(l) => buff.push_str(&l.text()),
GreenNode::Branch(b) => { GreenNode::Branch(b) => b.children().iter().for_each(|child| go(child, buff)),
b.children().iter().for_each(|child| go(child, buff))
}
} }
} }
} }
@ -71,7 +72,6 @@ impl GreenNodeBuilder {
} }
} }
#[test] #[test]
fn assert_send_sync() { fn assert_send_sync() {
fn f<T: Send + Sync>() {} fn f<T: Send + Sync>() {}
@ -80,14 +80,8 @@ fn assert_send_sync() {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub(crate) enum GreenLeaf { pub(crate) enum GreenLeaf {
Whitespace { Whitespace { newlines: u8, spaces: u8 },
newlines: u8, Token { kind: SyntaxKind, text: Arc<str> },
spaces: u8,
},
Token {
kind: SyntaxKind,
text: Arc<str>,
},
} }
impl GreenLeaf { impl GreenLeaf {
@ -96,10 +90,16 @@ impl GreenLeaf {
let newlines = text.bytes().take_while(|&b| b == b'\n').count(); let newlines = text.bytes().take_while(|&b| b == b'\n').count();
let spaces = text[newlines..].bytes().take_while(|&b| b == b' ').count(); let spaces = text[newlines..].bytes().take_while(|&b| b == b' ').count();
if newlines + spaces == text.len() && newlines <= N_NEWLINES && spaces <= N_SPACES { if newlines + spaces == text.len() && newlines <= N_NEWLINES && spaces <= N_SPACES {
return GreenLeaf::Whitespace { newlines: newlines as u8, spaces: spaces as u8 }; return GreenLeaf::Whitespace {
newlines: newlines as u8,
spaces: spaces as u8,
};
} }
} }
GreenLeaf::Token { kind, text: text.to_owned().into_boxed_str().into() } GreenLeaf::Token {
kind,
text: text.to_owned().into_boxed_str().into(),
}
} }
pub(crate) fn kind(&self) -> SyntaxKind { pub(crate) fn kind(&self) -> SyntaxKind {
@ -141,7 +141,11 @@ pub(crate) struct GreenBranch {
impl GreenBranch { impl GreenBranch {
fn new(kind: SyntaxKind, children: Vec<GreenNode>) -> GreenBranch { fn new(kind: SyntaxKind, children: Vec<GreenNode>) -> GreenBranch {
let text_len = children.iter().map(|x| x.text_len()).sum::<TextUnit>(); let text_len = children.iter().map(|x| x.text_len()).sum::<TextUnit>();
GreenBranch { text_len, kind, children } GreenBranch {
text_len,
kind,
children,
}
} }
pub fn kind(&self) -> SyntaxKind { pub fn kind(&self) -> SyntaxKind {
@ -156,4 +160,3 @@ impl GreenBranch {
self.children.as_slice() self.children.as_slice()
} }
} }

View file

@ -1,12 +1,12 @@
mod builder;
mod green; mod green;
mod red; mod red;
mod syntax; mod syntax;
mod builder;
pub use self::syntax::{SyntaxNode, SyntaxNodeRef};
pub(crate) use self::{ pub(crate) use self::{
builder::GreenBuilder,
green::{GreenNode, GreenNodeBuilder}, green::{GreenNode, GreenNodeBuilder},
red::RedNode, red::RedNode,
syntax::{SyntaxError, SyntaxRoot}, syntax::{SyntaxError, SyntaxRoot},
builder::GreenBuilder,
}; };
pub use self::syntax::{SyntaxNode, SyntaxNodeRef};

View file

@ -1,11 +1,5 @@
use std::{ use std::{ptr, sync::RwLock};
ptr, use {yellow::GreenNode, TextUnit};
sync::RwLock,
};
use {
TextUnit,
yellow::GreenNode,
};
#[derive(Debug)] #[derive(Debug)]
pub(crate) struct RedNode { pub(crate) struct RedNode {
@ -22,9 +16,7 @@ struct ParentData {
} }
impl RedNode { impl RedNode {
pub fn new_root( pub fn new_root(green: GreenNode) -> RedNode {
green: GreenNode,
) -> RedNode {
RedNode::new(green, None) RedNode::new(green, None)
} }
@ -42,13 +34,14 @@ impl RedNode {
RedNode::new(green, Some(parent_data)) RedNode::new(green, Some(parent_data))
} }
fn new( fn new(green: GreenNode, parent: Option<ParentData>) -> RedNode {
green: GreenNode,
parent: Option<ParentData>,
) -> RedNode {
let n_children = green.children().len(); let n_children = green.children().len();
let children = (0..n_children).map(|_| None).collect(); let children = (0..n_children).map(|_| None).collect();
RedNode { green, parent, children: RwLock::new(children) } RedNode {
green,
parent,
children: RwLock::new(children),
}
} }
pub(crate) fn green(&self) -> &GreenNode { pub(crate) fn green(&self) -> &GreenNode {
@ -75,12 +68,15 @@ impl RedNode {
if children[idx].is_none() { if children[idx].is_none() {
let green_children = self.green.children(); let green_children = self.green.children();
let start_offset = self.start_offset() let start_offset = self.start_offset()
+ green_children[..idx].iter().map(|x| x.text_len()).sum::<TextUnit>(); + green_children[..idx]
let child = RedNode::new_child(green_children[idx].clone(), self.into(), start_offset, idx); .iter()
.map(|x| x.text_len())
.sum::<TextUnit>();
let child =
RedNode::new_child(green_children[idx].clone(), self.into(), start_offset, idx);
children[idx] = Some(child) children[idx] = Some(child)
} }
children[idx].as_ref().unwrap().into() children[idx].as_ref().unwrap().into()
} }
pub(crate) fn parent(&self) -> Option<ptr::NonNull<RedNode>> { pub(crate) fn parent(&self) -> Option<ptr::NonNull<RedNode>> {

View file

@ -1,17 +1,12 @@
use std::{ use std::{fmt, ops::Deref, ptr, sync::Arc};
fmt,
sync::Arc,
ptr,
ops::Deref,
};
use { use {
TextRange, TextUnit, yellow::{GreenNode, RedNode},
SyntaxKind::{self, *}, SyntaxKind::{self, *},
yellow::{RedNode, GreenNode}, TextRange, TextUnit,
}; };
pub trait TreeRoot: Deref<Target=SyntaxRoot> + Clone {} pub trait TreeRoot: Deref<Target = SyntaxRoot> + Clone {}
impl TreeRoot for Arc<SyntaxRoot> {} impl TreeRoot for Arc<SyntaxRoot> {}
impl<'a> TreeRoot for &'a SyntaxRoot {} impl<'a> TreeRoot for &'a SyntaxRoot {}
@ -50,7 +45,10 @@ impl SyntaxNode<Arc<SyntaxRoot>> {
pub(crate) fn new_owned(root: SyntaxRoot) -> Self { pub(crate) fn new_owned(root: SyntaxRoot) -> Self {
let root = Arc::new(root); let root = Arc::new(root);
let red_weak = ptr::NonNull::from(&root.red); let red_weak = ptr::NonNull::from(&root.red);
SyntaxNode { root, red: red_weak } SyntaxNode {
root,
red: red_weak,
}
} }
} }
@ -68,10 +66,7 @@ impl<ROOT: TreeRoot> SyntaxNode<ROOT> {
pub fn range(&self) -> TextRange { pub fn range(&self) -> TextRange {
let red = self.red(); let red = self.red();
TextRange::offset_len( TextRange::offset_len(red.start_offset(), red.green().text_len())
red.start_offset(),
red.green().text_len(),
)
} }
pub fn text(&self) -> String { pub fn text(&self) -> String {

View file

@ -1,8 +1,8 @@
extern crate libsyntax2; extern crate libsyntax2;
extern crate testutils; extern crate testutils;
use libsyntax2::{parse}; use libsyntax2::parse;
use libsyntax2::utils::{dump_tree_green}; use libsyntax2::utils::dump_tree_green;
use testutils::dir_tests; use testutils::dir_tests;
#[test] #[test]

View file

@ -1,8 +1,8 @@
extern crate difference; extern crate difference;
extern crate file; extern crate file;
use std::path::{Path, PathBuf};
use std::fs::read_dir; use std::fs::read_dir;
use std::path::{Path, PathBuf};
use difference::Changeset; use difference::Changeset;

View file

@ -1,18 +1,14 @@
extern crate clap; extern crate clap;
#[macro_use] #[macro_use]
extern crate failure; extern crate failure;
extern crate tera;
extern crate ron;
extern crate walkdir;
extern crate itertools; extern crate itertools;
extern crate ron;
extern crate tera;
extern crate walkdir;
use std::{
fs,
path::{Path},
collections::HashSet,
};
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use itertools::Itertools; use itertools::Itertools;
use std::{collections::HashSet, fs, path::Path};
type Result<T> = ::std::result::Result<T, failure::Error>; type Result<T> = ::std::result::Result<T, failure::Error>;
@ -29,7 +25,7 @@ fn main() -> Result<()> {
Arg::with_name("verify") Arg::with_name("verify")
.long("--verify") .long("--verify")
.help("Verify that generated code is up-to-date") .help("Verify that generated code is up-to-date")
.global(true) .global(true),
) )
.subcommand(SubCommand::with_name("gen-kinds")) .subcommand(SubCommand::with_name("gen-kinds"))
.subcommand(SubCommand::with_name("gen-tests")) .subcommand(SubCommand::with_name("gen-tests"))
@ -66,9 +62,8 @@ fn update(path: &Path, contents: &str, verify: bool) -> Result<()> {
fn get_kinds() -> Result<String> { fn get_kinds() -> Result<String> {
let grammar = grammar()?; let grammar = grammar()?;
let template = fs::read_to_string(SYNTAX_KINDS_TEMPLATE)?; let template = fs::read_to_string(SYNTAX_KINDS_TEMPLATE)?;
let ret = tera::Tera::one_off(&template, &grammar, false).map_err(|e| { let ret = tera::Tera::one_off(&template, &grammar, false)
format_err!("template error: {}", e) .map_err(|e| format_err!("template error: {}", e))?;
})?;
Ok(ret) Ok(ret)
} }
@ -142,7 +137,8 @@ fn tests_from_dir(dir: &Path) -> Result<HashSet<Test>> {
fn collect_tests(s: &str) -> Vec<Test> { fn collect_tests(s: &str) -> Vec<Test> {
let mut res = vec![]; let mut res = vec![];
let prefix = "// "; let prefix = "// ";
let comment_blocks = s.lines() let comment_blocks = s
.lines()
.map(str::trim_left) .map(str::trim_left)
.group_by(|line| line.starts_with(prefix)); .group_by(|line| line.starts_with(prefix));
@ -181,4 +177,3 @@ fn existing_tests(dir: &Path) -> Result<HashSet<Test>> {
} }
Ok(res) Ok(res)
} }