This commit is contained in:
Aleksey Kladov 2018-07-31 15:40:40 +03:00
parent 2a2815266b
commit 9ce7e81102
19 changed files with 94 additions and 65 deletions

View file

@ -4,8 +4,8 @@ extern crate failure;
extern crate libsyntax2; extern crate libsyntax2;
extern crate tools; extern crate tools;
use std::{fs, path::Path, io::Read};
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use std::{fs, io::Read, path::Path};
use tools::collect_tests; use tools::collect_tests;
type Result<T> = ::std::result::Result<T, failure::Error>; type Result<T> = ::std::result::Result<T, failure::Error>;
@ -15,8 +15,18 @@ fn main() -> Result<()> {
.setting(clap::AppSettings::SubcommandRequiredElseHelp) .setting(clap::AppSettings::SubcommandRequiredElseHelp)
.subcommand( .subcommand(
SubCommand::with_name("render-test") SubCommand::with_name("render-test")
.arg(Arg::with_name("line").long("--line").required(true).takes_value(true)) .arg(
.arg(Arg::with_name("file").long("--file").required(true).takes_value(true)) Arg::with_name("line")
.long("--line")
.required(true)
.takes_value(true),
)
.arg(
Arg::with_name("file")
.long("--file")
.required(true)
.takes_value(true),
),
) )
.subcommand(SubCommand::with_name("parse")) .subcommand(SubCommand::with_name("parse"))
.get_matches(); .get_matches();
@ -24,7 +34,7 @@ fn main() -> Result<()> {
("parse", _) => { ("parse", _) => {
let tree = parse()?; let tree = parse()?;
println!("{}", tree); println!("{}", tree);
}, }
("render-test", Some(matches)) => { ("render-test", Some(matches)) => {
let file = matches.value_of("file").unwrap(); let file = matches.value_of("file").unwrap();
let file = Path::new(file); let file = Path::new(file);
@ -36,7 +46,6 @@ fn main() -> Result<()> {
_ => unreachable!(), _ => unreachable!(),
} }
Ok(()) Ok(())
} }
fn parse() -> Result<String> { fn parse() -> Result<String> {

View file

@ -1,6 +1,6 @@
use SyntaxNodeRef; use SyntaxNodeRef;
pub fn preorder<'a>(root: SyntaxNodeRef<'a>) -> impl Iterator<Item=SyntaxNodeRef<'a>> { pub fn preorder<'a>(root: SyntaxNodeRef<'a>) -> impl Iterator<Item = SyntaxNodeRef<'a>> {
walk(root).filter_map(|event| match event { walk(root).filter_map(|event| match event {
WalkEvent::Enter(node) => Some(node), WalkEvent::Enter(node) => Some(node),
WalkEvent::Exit(_) => None, WalkEvent::Exit(_) => None,
@ -13,7 +13,7 @@ enum WalkEvent<'a> {
Exit(SyntaxNodeRef<'a>), Exit(SyntaxNodeRef<'a>),
} }
fn walk<'a>(root: SyntaxNodeRef<'a>) -> impl Iterator<Item=WalkEvent<'a>> { fn walk<'a>(root: SyntaxNodeRef<'a>) -> impl Iterator<Item = WalkEvent<'a>> {
let mut done = false; let mut done = false;
::itertools::unfold(WalkEvent::Enter(root), move |pos| { ::itertools::unfold(WalkEvent::Enter(root), move |pos| {
if done { if done {
@ -35,7 +35,7 @@ fn walk<'a>(root: SyntaxNodeRef<'a>) -> impl Iterator<Item=WalkEvent<'a>> {
None => match node.parent() { None => match node.parent() {
Some(node) => WalkEvent::Exit(node), Some(node) => WalkEvent::Exit(node),
None => WalkEvent::Exit(node), None => WalkEvent::Exit(node),
} },
} }
} }
} }
@ -43,4 +43,3 @@ fn walk<'a>(root: SyntaxNodeRef<'a>) -> impl Iterator<Item=WalkEvent<'a>> {
Some(res) Some(res)
}) })
} }

View file

@ -1,14 +1,16 @@
use std::sync::Arc; use std::sync::Arc;
use {SyntaxNode, TreeRoot, SyntaxRoot}; use {SyntaxNode, SyntaxRoot, TreeRoot};
#[derive(Debug)] #[derive(Debug)]
pub struct File<R: TreeRoot = Arc<SyntaxRoot>> { pub struct File<R: TreeRoot = Arc<SyntaxRoot>> {
syntax: SyntaxNode<R> syntax: SyntaxNode<R>,
} }
impl File<Arc<SyntaxRoot>> { impl File<Arc<SyntaxRoot>> {
pub fn parse(text: &str) -> Self { pub fn parse(text: &str) -> Self {
File { syntax: ::parse(text.to_owned()) } File {
syntax: ::parse(text.to_owned()),
}
} }
} }

View file

@ -20,25 +20,25 @@
#![allow(missing_docs)] #![allow(missing_docs)]
//#![warn(unreachable_pub)] // rust-lang/rust#47816 //#![warn(unreachable_pub)] // rust-lang/rust#47816
extern crate itertools;
extern crate text_unit; extern crate text_unit;
extern crate unicode_xid; extern crate unicode_xid;
extern crate itertools;
pub mod algo;
pub mod ast;
mod lexer; mod lexer;
mod parser; mod parser;
mod syntax_kinds; mod syntax_kinds;
mod yellow;
/// Utilities for simple uses of the parser. /// Utilities for simple uses of the parser.
pub mod utils; pub mod utils;
pub mod ast; mod yellow;
pub mod algo;
pub use { pub use {
ast::File,
lexer::{tokenize, Token}, lexer::{tokenize, Token},
syntax_kinds::SyntaxKind, syntax_kinds::SyntaxKind,
text_unit::{TextRange, TextUnit}, text_unit::{TextRange, TextUnit},
yellow::{SyntaxNode, SyntaxNodeRef, TreeRoot, SyntaxRoot}, yellow::{SyntaxNode, SyntaxNodeRef, SyntaxRoot, TreeRoot},
ast::File,
}; };
pub(crate) use yellow::SyntaxError; pub(crate) use yellow::SyntaxError;
@ -47,4 +47,3 @@ pub fn parse(text: String) -> SyntaxNode {
let tokens = tokenize(&text); let tokens = tokenize(&text);
parser::parse::<yellow::GreenBuilder>(text, &tokens) parser::parse::<yellow::GreenBuilder>(text, &tokens)
} }

View file

@ -39,7 +39,7 @@ pub(super) fn expr(p: &mut Parser) {
fn prefix_expr(p: &mut Parser) -> Option<CompletedMarker> { fn prefix_expr(p: &mut Parser) -> Option<CompletedMarker> {
match p.current() { match p.current() {
AMPERSAND => Some(ref_expr(p)), AMPERSAND => Some(ref_expr(p)),
_ => atom_expr(p) _ => atom_expr(p),
} }
} }

View file

@ -218,7 +218,6 @@ fn extern_block(p: &mut Parser) {
p.expect(R_CURLY); p.expect(R_CURLY);
} }
fn fn_item(p: &mut Parser) { fn fn_item(p: &mut Parser) {
assert!(p.at(FN_KW)); assert!(p.at(FN_KW));
p.bump(); p.bump();

View file

@ -8,7 +8,6 @@ pub(super) fn trait_item(p: &mut Parser) {
p.expect(R_CURLY); p.expect(R_CURLY);
} }
// test impl_item // test impl_item
// impl Foo {} // impl Foo {}
pub(super) fn impl_item(p: &mut Parser) { pub(super) fn impl_item(p: &mut Parser) {

View file

@ -26,12 +26,15 @@ mod expressions;
mod items; mod items;
mod paths; mod paths;
mod patterns; mod patterns;
mod type_params;
mod type_args; mod type_args;
mod type_params;
mod types; mod types;
use { use {
parser::{parser::{Parser, CompletedMarker}, token_set::TokenSet}, parser::{
parser::{CompletedMarker, Parser},
token_set::TokenSet,
},
SyntaxKind::{self, *}, SyntaxKind::{self, *},
}; };

View file

@ -20,7 +20,11 @@ pub(super) fn expr_path(p: &mut Parser) {
} }
#[derive(Clone, Copy, Eq, PartialEq)] #[derive(Clone, Copy, Eq, PartialEq)]
enum Mode { Use, Type, Expr } enum Mode {
Use,
Type,
Expr,
}
fn path(p: &mut Parser, mode: Mode) { fn path(p: &mut Parser, mode: Mode) {
if !is_path_start(p) { if !is_path_start(p) {
@ -55,7 +59,7 @@ fn path_segment(p: &mut Parser, mode: Mode, first: bool) {
IDENT => { IDENT => {
name_ref(p); name_ref(p);
path_generic_args(p, mode); path_generic_args(p, mode);
}, }
SELF_KW | SUPER_KW => p.bump(), SELF_KW | SUPER_KW => p.bump(),
_ => { _ => {
p.error("expected identifier"); p.error("expected identifier");

View file

@ -43,11 +43,8 @@ fn ref_pat(p: &mut Parser) {
// } // }
fn bind_pat(p: &mut Parser) { fn bind_pat(p: &mut Parser) {
let m = p.start(); let m = p.start();
if p.eat(REF_KW) { p.eat(REF_KW);
p.eat(MUT_KW); p.eat(MUT_KW);
} else {
p.eat(MUT_KW);
}
name(p); name(p);
if p.eat(AT) { if p.eat(AT) {
pattern(p); pattern(p);

View file

@ -12,7 +12,7 @@ pub(super) fn list(p: &mut Parser, colon_colon_required: bool) {
m = p.start(); m = p.start();
p.bump(); p.bump();
} }
_ => return _ => return,
}; };
while !p.at(EOF) && !p.at(R_ANGLE) { while !p.at(EOF) && !p.at(R_ANGLE) {

View file

@ -16,7 +16,7 @@ pub fn dump_tree(syntax: &SyntaxNode) -> String {
errors: &mut BTreeSet<SyntaxError>, errors: &mut BTreeSet<SyntaxError>,
) { ) {
buff.push_str(&String::from(" ").repeat(level)); buff.push_str(&String::from(" ").repeat(level));
write!(buff, "{:?}\n", node).unwrap(); writeln!(buff, "{:?}", node).unwrap();
let my_errors: Vec<_> = errors let my_errors: Vec<_> = errors
.iter() .iter()
.filter(|e| e.offset == node.range().start()) .filter(|e| e.offset == node.range().start())
@ -25,7 +25,7 @@ pub fn dump_tree(syntax: &SyntaxNode) -> String {
for err in my_errors { for err in my_errors {
errors.remove(&err); errors.remove(&err);
buff.push_str(&String::from(" ").repeat(level)); buff.push_str(&String::from(" ").repeat(level));
write!(buff, "err: `{}`\n", err.message).unwrap(); writeln!(buff, "err: `{}`", err.message).unwrap();
} }
for child in node.children() { for child in node.children() {
@ -40,7 +40,7 @@ pub fn dump_tree(syntax: &SyntaxNode) -> String {
for err in my_errors { for err in my_errors {
errors.remove(&err); errors.remove(&err);
buff.push_str(&String::from(" ").repeat(level)); buff.push_str(&String::from(" ").repeat(level));
write!(buff, "err: `{}`\n", err.message).unwrap(); writeln!(buff, "err: `{}`", err.message).unwrap();
} }
} }
} }

View file

@ -80,8 +80,14 @@ fn assert_send_sync() {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub(crate) enum GreenLeaf { pub(crate) enum GreenLeaf {
Whitespace { newlines: u8, spaces: u8 }, Whitespace {
Token { kind: SyntaxKind, text: Option<Arc<str>> }, newlines: u8,
spaces: u8,
},
Token {
kind: SyntaxKind,
text: Option<Arc<str>>,
},
} }
impl GreenLeaf { impl GreenLeaf {
@ -121,7 +127,7 @@ impl GreenLeaf {
assert!(newlines <= N_NEWLINES && spaces <= N_SPACES); assert!(newlines <= N_NEWLINES && spaces <= N_SPACES);
&WS[N_NEWLINES - newlines..N_NEWLINES + spaces] &WS[N_NEWLINES - newlines..N_NEWLINES + spaces]
} }
GreenLeaf::Token { kind, text, } => match text { GreenLeaf::Token { kind, text } => match text {
None => kind.static_text().unwrap(), None => kind.static_text().unwrap(),
Some(t) => t, Some(t) => t,
}, },

View file

@ -8,5 +8,5 @@ pub(crate) use self::{
builder::GreenBuilder, builder::GreenBuilder,
green::{GreenNode, GreenNodeBuilder}, green::{GreenNode, GreenNodeBuilder},
red::RedNode, red::RedNode,
syntax::{SyntaxError}, syntax::SyntaxError,
}; };

View file

@ -36,7 +36,8 @@ impl RedNode {
fn new(green: GreenNode, parent: Option<ParentData>) -> RedNode { fn new(green: GreenNode, parent: Option<ParentData>) -> RedNode {
let n_children = green.children().len(); let n_children = green.children().len();
let children = (0..n_children).map(|_| None) let children = (0..n_children)
.map(|_| None)
.collect::<Vec<_>>() .collect::<Vec<_>>()
.into_boxed_slice(); .into_boxed_slice();
RedNode { RedNode {
@ -63,7 +64,7 @@ impl RedNode {
pub(crate) fn get_child(&self, idx: usize) -> Option<ptr::NonNull<RedNode>> { pub(crate) fn get_child(&self, idx: usize) -> Option<ptr::NonNull<RedNode>> {
if idx >= self.n_children() { if idx >= self.n_children() {
return None return None;
} }
match &self.children.read().unwrap()[idx] { match &self.children.read().unwrap()[idx] {
Some(child) => return Some(child.into()), Some(child) => return Some(child.into()),

View file

@ -6,7 +6,7 @@ use {
TextRange, TextUnit, TextRange, TextUnit,
}; };
pub trait TreeRoot: Deref<Target=SyntaxRoot> + Clone {} pub trait TreeRoot: Deref<Target = SyntaxRoot> + Clone {}
impl TreeRoot for Arc<SyntaxRoot> {} impl TreeRoot for Arc<SyntaxRoot> {}
@ -83,14 +83,12 @@ impl<R: TreeRoot> SyntaxNode<R> {
self.red().green().text() self.red().green().text()
} }
pub fn children<'a>(&'a self) -> impl Iterator<Item=SyntaxNode<R>> + 'a { pub fn children<'a>(&'a self) -> impl Iterator<Item = SyntaxNode<R>> + 'a {
let red = self.red(); let red = self.red();
let n_children = red.n_children(); let n_children = red.n_children();
(0..n_children).map(move |i| { (0..n_children).map(move |i| SyntaxNode {
SyntaxNode { root: self.root.clone(),
root: self.root.clone(), red: red.get_child(i).unwrap(),
red: red.get_child(i).unwrap(),
}
}) })
} }

View file

@ -2,7 +2,7 @@ extern crate difference;
use std::{ use std::{
fs, fs,
path::{Path, PathBuf} path::{Path, PathBuf},
}; };
use difference::Changeset; use difference::Changeset;

View file

@ -1,7 +1,7 @@
extern crate itertools; extern crate itertools;
use std::hash;
use itertools::Itertools; use itertools::Itertools;
use std::hash;
#[derive(Debug)] #[derive(Debug)]
pub struct Test { pub struct Test {
@ -28,18 +28,17 @@ pub fn collect_tests(s: &str) -> Vec<(usize, Test)> {
match block.next() { match block.next() {
Some((idx, line)) if line.starts_with("test ") => { Some((idx, line)) if line.starts_with("test ") => {
break (idx, line["test ".len()..].to_string()) break (idx, line["test ".len()..].to_string())
}, }
Some(_) => (), Some(_) => (),
None => continue 'outer, None => continue 'outer,
} }
}; };
let text: String = itertools::join( let text: String = itertools::join(
block.map(|(_, line)| line) block.map(|(_, line)| line).chain(::std::iter::once("")),
.chain(::std::iter::once("")), "\n",
"\n"
); );
assert!(!text.trim().is_empty() && text.ends_with("\n")); assert!(!text.trim().is_empty() && text.ends_with("\n"));
res.push((start_line, Test {name, text })) res.push((start_line, Test { name, text }))
} }
res res
} }

View file

@ -3,13 +3,17 @@ extern crate clap;
extern crate failure; extern crate failure;
extern crate ron; extern crate ron;
extern crate tera; extern crate tera;
extern crate walkdir;
extern crate tools; extern crate tools;
extern crate walkdir;
#[macro_use] #[macro_use]
extern crate commandspec; extern crate commandspec;
use std::{collections::{HashMap}, fs, path::{Path, PathBuf}};
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use std::{
collections::HashMap,
fs,
path::{Path, PathBuf},
};
use tools::{collect_tests, Test}; use tools::{collect_tests, Test};
type Result<T> = ::std::result::Result<T, failure::Error>; type Result<T> = ::std::result::Result<T, failure::Error>;
@ -71,7 +75,8 @@ fn get_kinds() -> Result<String> {
tera.add_raw_template("grammar", &template) tera.add_raw_template("grammar", &template)
.map_err(|e| format_err!("template error: {:?}", e))?; .map_err(|e| format_err!("template error: {:?}", e))?;
tera.register_global_function("concat", Box::new(concat)); tera.register_global_function("concat", Box::new(concat));
let ret = tera.render("grammar", &grammar) let ret = tera
.render("grammar", &grammar)
.map_err(|e| format_err!("template error: {:?}", e))?; .map_err(|e| format_err!("template error: {:?}", e))?;
return Ok(ret); return Ok(ret);
@ -157,7 +162,10 @@ fn existing_tests(dir: &Path) -> Result<HashMap<String, (PathBuf, Test)>> {
file_name[5..file_name.len() - 3].to_string() file_name[5..file_name.len() - 3].to_string()
}; };
let text = fs::read_to_string(&path)?; let text = fs::read_to_string(&path)?;
let test = Test { name: name.clone(), text }; let test = Test {
name: name.clone(),
text,
};
match res.insert(name, (path, test)) { match res.insert(name, (path, test)) {
Some(old) => println!("Duplicate test: {:?}", old), Some(old) => println!("Duplicate test: {:?}", old),
None => (), None => (),
@ -167,17 +175,23 @@ fn existing_tests(dir: &Path) -> Result<HashMap<String, (PathBuf, Test)>> {
} }
fn install_code_extension() -> Result<()> { fn install_code_extension() -> Result<()> {
execute!(r" execute!(
r"
cd code cd code
npm install npm install
")?; "
execute!(r" )?;
execute!(
r"
cd code cd code
./node_modules/vsce/out/vsce package ./node_modules/vsce/out/vsce package
")?; "
execute!(r" )?;
execute!(
r"
cd code cd code
code --install-extension ./libsyntax-rust-0.0.1.vsix code --install-extension ./libsyntax-rust-0.0.1.vsix
")?; "
)?;
Ok(()) Ok(())
} }