Merge #43

43: Ctx r=matklad a=matklad

grammar.ron

@@ -27,6 +27,11 @@ Grammar(
         "mut",
         "unsafe",
     ],
+    contextual_keywords: [
+        "auto",
+        "default",
+        "union",
+    ],
     tokens: [
         "ERROR",
         "IDENT",

src/parser/event.rs

@@ -1,4 +1,4 @@
-use {File, FileBuilder, Sink, SyntaxKind, Token};
+use {File, FileBuilder, Sink, SyntaxKind, TextUnit, Token};
 use syntax_kinds::TOMBSTONE;
 use super::is_insignificant;
 
@@ -120,19 +120,26 @@ pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> Fil
                 builder.finish_internal()
             }
             &Event::Token {
-                kind: _,
+                kind,
                 mut n_raw_tokens,
-            } => loop {
+            } => {
+                // FIXME: currently, we attach whitespace to some random node
+                // this should be done in a sensible manner instead
+                loop {
                     let token = tokens[idx];
                     if !is_insignificant(token.kind) {
-                    n_raw_tokens -= 1;
-                }
-                idx += 1;
-                builder.leaf(token.kind, token.len);
-                if n_raw_tokens == 0 {
                         break;
                     }
-            },
+                    builder.leaf(token.kind, token.len);
+                    idx += 1
+                }
+                let mut len = TextUnit::new(0);
+                for _ in 0..n_raw_tokens {
+                    len += tokens[idx].len;
+                    idx += 1;
+                }
+                builder.leaf(kind, len);
+            }
             &Event::Error { ref message } => builder.error().message(message.clone()).emit(),
         }
     }

src/parser/grammar/items/mod.rs

@@ -81,22 +81,76 @@ fn item(p: &mut Parser) {
                 CONST_ITEM
             }
         },
-        // TODO: auto trait
+        UNSAFE_KW => {
+            p.bump();
+            let la = p.nth(1);
+            match p.current() {
                 // test unsafe_trait
                 // unsafe trait T {}
-        UNSAFE_KW if la == TRAIT_KW => {
+                TRAIT_KW => {
-            p.bump();
                     traits::trait_item(p);
                     TRAIT_ITEM
                 }
-        // TODO: default impl
+
+                // test unsafe_auto_trait
+                // unsafe auto trait T {}
+                IDENT if p.at_kw("auto") && la == TRAIT_KW => {
+                    p.bump_remap(AUTO_KW);
+                    traits::trait_item(p);
+                    TRAIT_ITEM
+                }
+
                 // test unsafe_impl
                 // unsafe impl Foo {}
-        UNSAFE_KW if la == IMPL_KW => {
+                IMPL_KW => {
-            p.bump();
                     traits::impl_item(p);
                     IMPL_ITEM
                 }
+
+                // test unsafe_default_impl
+                // unsafe default impl Foo {}
+                IDENT if p.at_kw("default") && la == IMPL_KW => {
+                    p.bump_remap(DEFAULT_KW);
+                    traits::impl_item(p);
+                    IMPL_ITEM
+                }
+
+                // test unsafe_extern_fn
+                // unsafe extern "C" fn foo() {}
+                EXTERN_KW => {
+                    abi(p);
+                    if !p.at(FN_KW) {
+                        item.abandon(p);
+                        p.error().message("expected function").emit();
+                        return;
+                    }
+                    fn_item(p);
+                    FN_ITEM
+                }
+
+                // test unsafe_fn
+                // unsafe fn foo() {}
+                FN_KW => {
+                    fn_item(p);
+                    FN_ITEM
+                }
+
+                t => {
+                    item.abandon(p);
+                    let message = "expected `trait`, `impl` or `fn`";
+
+                    // test unsafe_block_in_mod
+                    // fn foo(){} unsafe { } fn bar(){}
+                    if t == L_CURLY {
+                        error_block(p, message);
+                    } else {
+                        p.error().message(message).emit();
+                    }
+                    return;
+                }
+            }
+        }
+
         MOD_KW => {
             mod_item(p);
             MOD_ITEM

src/parser/input.rs

@@ -46,9 +46,7 @@ impl<'t> ParserInput<'t> {
         if !(idx < self.tokens.len()) {
             return "";
         }
-        let start_offset = self.start_offsets[idx];
+        let range = TextRange::from_len(self.start_offsets[idx], self.tokens[idx].len);
-        let end_offset = self.tokens[idx].len;
-        let range = TextRange::from_to(start_offset, end_offset);
         &self.text[range]
     }
 }

src/parser/parser.rs

@@ -145,10 +145,26 @@ impl<'t> Parser<'t> {
         });
     }
 
+    pub(crate) fn bump_remap(&mut self, kind: SyntaxKind) {
+        if self.current() == EOF {
+            // TODO: panic!?
+            return;
+        }
+        self.pos += 1;
+        self.event(Event::Token {
+            kind,
+            n_raw_tokens: 1,
+        });
+    }
+
     pub(crate) fn nth(&self, n: u32) -> SyntaxKind {
         self.inp.kind(self.pos + n)
     }
 
+    pub(crate) fn at_kw(&self, t: &str) -> bool {
+        self.inp.text(self.pos) == t
+    }
+
     pub(crate) fn current(&self) -> SyntaxKind {
         self.nth(0)
     }

src/syntax_kinds.rs

@@ -6,32 +6,6 @@ use tree::SyntaxInfo;
 /// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum SyntaxKind {
-    USE_KW,
-    FN_KW,
-    STRUCT_KW,
-    ENUM_KW,
-    TRAIT_KW,
-    IMPL_KW,
-    TRUE_KW,
-    FALSE_KW,
-    AS_KW,
-    EXTERN_KW,
-    CRATE_KW,
-    MOD_KW,
-    PUB_KW,
-    SELF_KW,
-    SUPER_KW,
-    IN_KW,
-    WHERE_KW,
-    FOR_KW,
-    LOOP_KW,
-    WHILE_KW,
-    IF_KW,
-    MATCH_KW,
-    CONST_KW,
-    STATIC_KW,
-    MUT_KW,
-    UNSAFE_KW,
     ERROR,
     IDENT,
     UNDERSCORE,
@@ -83,6 +57,35 @@ pub enum SyntaxKind {
     COMMENT,
     DOC_COMMENT,
     SHEBANG,
+    USE_KW,
+    FN_KW,
+    STRUCT_KW,
+    ENUM_KW,
+    TRAIT_KW,
+    IMPL_KW,
+    TRUE_KW,
+    FALSE_KW,
+    AS_KW,
+    EXTERN_KW,
+    CRATE_KW,
+    MOD_KW,
+    PUB_KW,
+    SELF_KW,
+    SUPER_KW,
+    IN_KW,
+    WHERE_KW,
+    FOR_KW,
+    LOOP_KW,
+    WHILE_KW,
+    IF_KW,
+    MATCH_KW,
+    CONST_KW,
+    STATIC_KW,
+    MUT_KW,
+    UNSAFE_KW,
+    AUTO_KW,
+    DEFAULT_KW,
+    UNION_KW,
     FILE,
     STRUCT_ITEM,
     ENUM_ITEM,
@@ -123,32 +126,6 @@ pub(crate) use self::SyntaxKind::*;
 impl SyntaxKind {
     pub(crate) fn info(self) -> &'static SyntaxInfo {
         match self {
-            USE_KW => &SyntaxInfo { name: "USE_KW" },
-            FN_KW => &SyntaxInfo { name: "FN_KW" },
-            STRUCT_KW => &SyntaxInfo { name: "STRUCT_KW" },
-            ENUM_KW => &SyntaxInfo { name: "ENUM_KW" },
-            TRAIT_KW => &SyntaxInfo { name: "TRAIT_KW" },
-            IMPL_KW => &SyntaxInfo { name: "IMPL_KW" },
-            TRUE_KW => &SyntaxInfo { name: "TRUE_KW" },
-            FALSE_KW => &SyntaxInfo { name: "FALSE_KW" },
-            AS_KW => &SyntaxInfo { name: "AS_KW" },
-            EXTERN_KW => &SyntaxInfo { name: "EXTERN_KW" },
-            CRATE_KW => &SyntaxInfo { name: "CRATE_KW" },
-            MOD_KW => &SyntaxInfo { name: "MOD_KW" },
-            PUB_KW => &SyntaxInfo { name: "PUB_KW" },
-            SELF_KW => &SyntaxInfo { name: "SELF_KW" },
-            SUPER_KW => &SyntaxInfo { name: "SUPER_KW" },
-            IN_KW => &SyntaxInfo { name: "IN_KW" },
-            WHERE_KW => &SyntaxInfo { name: "WHERE_KW" },
-            FOR_KW => &SyntaxInfo { name: "FOR_KW" },
-            LOOP_KW => &SyntaxInfo { name: "LOOP_KW" },
-            WHILE_KW => &SyntaxInfo { name: "WHILE_KW" },
-            IF_KW => &SyntaxInfo { name: "IF_KW" },
-            MATCH_KW => &SyntaxInfo { name: "MATCH_KW" },
-            CONST_KW => &SyntaxInfo { name: "CONST_KW" },
-            STATIC_KW => &SyntaxInfo { name: "STATIC_KW" },
-            MUT_KW => &SyntaxInfo { name: "MUT_KW" },
-            UNSAFE_KW => &SyntaxInfo { name: "UNSAFE_KW" },
             ERROR => &SyntaxInfo { name: "ERROR" },
             IDENT => &SyntaxInfo { name: "IDENT" },
             UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" },
@@ -200,6 +177,35 @@ impl SyntaxKind {
             COMMENT => &SyntaxInfo { name: "COMMENT" },
             DOC_COMMENT => &SyntaxInfo { name: "DOC_COMMENT" },
             SHEBANG => &SyntaxInfo { name: "SHEBANG" },
+            USE_KW => &SyntaxInfo { name: "USE_KW" },
+            FN_KW => &SyntaxInfo { name: "FN_KW" },
+            STRUCT_KW => &SyntaxInfo { name: "STRUCT_KW" },
+            ENUM_KW => &SyntaxInfo { name: "ENUM_KW" },
+            TRAIT_KW => &SyntaxInfo { name: "TRAIT_KW" },
+            IMPL_KW => &SyntaxInfo { name: "IMPL_KW" },
+            TRUE_KW => &SyntaxInfo { name: "TRUE_KW" },
+            FALSE_KW => &SyntaxInfo { name: "FALSE_KW" },
+            AS_KW => &SyntaxInfo { name: "AS_KW" },
+            EXTERN_KW => &SyntaxInfo { name: "EXTERN_KW" },
+            CRATE_KW => &SyntaxInfo { name: "CRATE_KW" },
+            MOD_KW => &SyntaxInfo { name: "MOD_KW" },
+            PUB_KW => &SyntaxInfo { name: "PUB_KW" },
+            SELF_KW => &SyntaxInfo { name: "SELF_KW" },
+            SUPER_KW => &SyntaxInfo { name: "SUPER_KW" },
+            IN_KW => &SyntaxInfo { name: "IN_KW" },
+            WHERE_KW => &SyntaxInfo { name: "WHERE_KW" },
+            FOR_KW => &SyntaxInfo { name: "FOR_KW" },
+            LOOP_KW => &SyntaxInfo { name: "LOOP_KW" },
+            WHILE_KW => &SyntaxInfo { name: "WHILE_KW" },
+            IF_KW => &SyntaxInfo { name: "IF_KW" },
+            MATCH_KW => &SyntaxInfo { name: "MATCH_KW" },
+            CONST_KW => &SyntaxInfo { name: "CONST_KW" },
+            STATIC_KW => &SyntaxInfo { name: "STATIC_KW" },
+            MUT_KW => &SyntaxInfo { name: "MUT_KW" },
+            UNSAFE_KW => &SyntaxInfo { name: "UNSAFE_KW" },
+            AUTO_KW => &SyntaxInfo { name: "AUTO_KW" },
+            DEFAULT_KW => &SyntaxInfo { name: "DEFAULT_KW" },
+            UNION_KW => &SyntaxInfo { name: "UNION_KW" },
             FILE => &SyntaxInfo { name: "FILE" },
             STRUCT_ITEM => &SyntaxInfo { name: "STRUCT_ITEM" },
             ENUM_ITEM => &SyntaxInfo { name: "ENUM_ITEM" },

tests/data/parser/inline/0009_unsafe_auto_trait.rs

@@ -0,0 +1 @@
+unsafe auto trait T {}

tests/data/parser/inline/0009_unsafe_auto_trait.txt

@@ -0,0 +1,13 @@
+FILE@[0; 23)
+  TRAIT_ITEM@[0; 23)
+    UNSAFE_KW@[0; 6)
+    WHITESPACE@[6; 7)
+    AUTO_KW@[7; 11)
+    WHITESPACE@[11; 12)
+    TRAIT_KW@[12; 17)
+    WHITESPACE@[17; 18)
+    IDENT@[18; 19) "T"
+    WHITESPACE@[19; 20)
+    L_CURLY@[20; 21)
+    R_CURLY@[21; 22)
+    WHITESPACE@[22; 23)

tests/data/parser/inline/0010_unsafe_default_impl.rs

@@ -0,0 +1 @@
+unsafe default impl Foo {}

tests/data/parser/inline/0010_unsafe_default_impl.txt

@@ -0,0 +1,13 @@
+FILE@[0; 27)
+  IMPL_ITEM@[0; 27)
+    UNSAFE_KW@[0; 6)
+    WHITESPACE@[6; 7)
+    DEFAULT_KW@[7; 14)
+    WHITESPACE@[14; 15)
+    IMPL_KW@[15; 19)
+    WHITESPACE@[19; 20)
+    IDENT@[20; 23) "Foo"
+    WHITESPACE@[23; 24)
+    L_CURLY@[24; 25)
+    R_CURLY@[25; 26)
+    WHITESPACE@[26; 27)

tests/data/parser/inline/0011_unsafe_fn.rs

@@ -0,0 +1 @@
+unsafe fn foo() {}

tests/data/parser/inline/0011_unsafe_fn.txt

@@ -0,0 +1,13 @@
+FILE@[0; 19)
+  FN_ITEM@[0; 19)
+    UNSAFE_KW@[0; 6)
+    WHITESPACE@[6; 7)
+    FN_KW@[7; 9)
+    WHITESPACE@[9; 10)
+    IDENT@[10; 13) "foo"
+    L_PAREN@[13; 14)
+    R_PAREN@[14; 15)
+    WHITESPACE@[15; 16)
+    L_CURLY@[16; 17)
+    R_CURLY@[17; 18)
+    WHITESPACE@[18; 19)

tests/data/parser/inline/0012_unsafe_extern_fn.rs

@@ -0,0 +1 @@
+unsafe extern "C" fn foo() {}

tests/data/parser/inline/0012_unsafe_extern_fn.txt

@@ -0,0 +1,18 @@
+FILE@[0; 30)
+  FN_ITEM@[0; 30)
+    UNSAFE_KW@[0; 6)
+    ABI@[6; 18)
+      WHITESPACE@[6; 7)
+      EXTERN_KW@[7; 13)
+      WHITESPACE@[13; 14)
+      STRING@[14; 17)
+      WHITESPACE@[17; 18)
+    FN_KW@[18; 20)
+    WHITESPACE@[20; 21)
+    IDENT@[21; 24) "foo"
+    L_PAREN@[24; 25)
+    R_PAREN@[25; 26)
+    WHITESPACE@[26; 27)
+    L_CURLY@[27; 28)
+    R_CURLY@[28; 29)
+    WHITESPACE@[29; 30)

tests/data/parser/inline/0013_unsafe_block_in_mod.rs

@@ -0,0 +1 @@
+fn foo(){} unsafe { } fn bar(){}

tests/data/parser/inline/0013_unsafe_block_in_mod.txt

@@ -0,0 +1,27 @@
+FILE@[0; 33)
+  FN_ITEM@[0; 11)
+    FN_KW@[0; 2)
+    WHITESPACE@[2; 3)
+    IDENT@[3; 6) "foo"
+    L_PAREN@[6; 7)
+    R_PAREN@[7; 8)
+    L_CURLY@[8; 9)
+    R_CURLY@[9; 10)
+    WHITESPACE@[10; 11)
+  UNSAFE_KW@[11; 17)
+  ERROR@[17; 22)
+  err: `expected `trait`, `impl` or `fn``
+    WHITESPACE@[17; 18)
+    L_CURLY@[18; 19)
+    WHITESPACE@[19; 20)
+    R_CURLY@[20; 21)
+    WHITESPACE@[21; 22)
+  FN_ITEM@[22; 33)
+    FN_KW@[22; 24)
+    WHITESPACE@[24; 25)
+    IDENT@[25; 28) "bar"
+    L_PAREN@[28; 29)
+    R_PAREN@[29; 30)
+    L_CURLY@[30; 31)
+    R_CURLY@[31; 32)
+    WHITESPACE@[32; 33)

tools/src/bin/gen.rs

@@ -20,6 +20,7 @@ fn main() {
 #[derive(Deserialize)]
 struct Grammar {
     keywords: Vec<String>,
+    contextual_keywords: Vec<String>,
     tokens: Vec<String>,
     nodes: Vec<String>,
 }
@@ -38,10 +39,11 @@ impl Grammar {
         acc.push_str("use tree::SyntaxInfo;\n");
         acc.push_str("\n");
 
-        let syntax_kinds: Vec<String> = self.keywords
+        let syntax_kinds: Vec<String> = self.tokens
             .iter()
-            .map(|kw| kw_token(kw))
+            .cloned()
-            .chain(self.tokens.iter().cloned())
+            .chain(self.keywords.iter().map(|kw| kw_token(kw)))
+            .chain(self.contextual_keywords.iter().map(|kw| kw_token(kw)))
             .chain(self.nodes.iter().cloned())
             .collect();
 
@@ -86,6 +88,7 @@ impl Grammar {
         // fn ident_to_keyword
         acc.push_str("pub(crate) fn ident_to_keyword(ident: &str) -> Option<SyntaxKind> {\n");
         acc.push_str("    match ident {\n");
+        // NB: no contextual_keywords here!
         for kw in self.keywords.iter() {
             write!(acc, "        {:?} => Some({}),\n", kw, kw_token(kw)).unwrap();
         }