Recover better for more delimited sequences

crates/hir-def/src/macro_expansion_tests/mbe/regression.rs

@@ -830,7 +830,6 @@ macro_rules! rgb_color {
 /* parse error: expected COMMA */
 /* parse error: expected R_ANGLE */
 /* parse error: expected SEMICOLON */
-/* parse error: expected SEMICOLON */
 /* parse error: expected expression, item or let statement */
 pub fn new() {
     let _ = 0as u32<<(8+8);
@@ -848,21 +847,21 @@ pub fn new() {
 //     BLOCK_EXPR@10..31
 //       STMT_LIST@10..31
 //         L_CURLY@10..11 "{"
-//         LET_STMT@11..27
+//         LET_STMT@11..28
 //           LET_KW@11..14 "let"
 //           WILDCARD_PAT@14..15
 //             UNDERSCORE@14..15 "_"
 //           EQ@15..16 "="
-//           CAST_EXPR@16..27
+//           CAST_EXPR@16..28
 //             LITERAL@16..17
 //               INT_NUMBER@16..17 "0"
 //             AS_KW@17..19 "as"
-//             PATH_TYPE@19..27
+//             PATH_TYPE@19..28
-//               PATH@19..27
+//               PATH@19..28
-//                 PATH_SEGMENT@19..27
+//                 PATH_SEGMENT@19..28
 //                   NAME_REF@19..22
 //                     IDENT@19..22 "u32"
-//                   GENERIC_ARG_LIST@22..27
+//                   GENERIC_ARG_LIST@22..28
 //                     L_ANGLE@22..23 "<"
 //                     TYPE_ARG@23..27
 //                       DYN_TRAIT_TYPE@23..27
@@ -877,9 +876,9 @@ pub fn new() {
 //                                     ERROR@25..26
 //                                       INT_NUMBER@25..26 "8"
 //                           PLUS@26..27 "+"
-//         EXPR_STMT@27..28
+//                     CONST_ARG@27..28
-//           LITERAL@27..28
+//                       LITERAL@27..28
-//             INT_NUMBER@27..28 "8"
+//                         INT_NUMBER@27..28 "8"
 //         ERROR@28..29
 //           R_PAREN@28..29 ")"
 //         SEMICOLON@29..30 ";"

crates/ide/src/syntax_highlighting/tests.rs

@@ -1126,5 +1126,5 @@ fn benchmark_syntax_highlighting_parser() {
             .filter(|it| it.highlight.tag == HlTag::Symbol(SymbolKind::Function))
             .count()
     };
-    assert_eq!(hash, 1609);
+    assert_eq!(hash, 1608);
 }

crates/parser/src/grammar.rs

@@ -200,6 +200,8 @@ impl BlockLike {
     }
 }
 
+const VISIBILITY_FIRST: TokenSet = TokenSet::new(&[T![pub], T![crate]]);
+
 fn opt_visibility(p: &mut Parser<'_>, in_tuple_field: bool) -> bool {
     match p.current() {
         T![pub] => {
@@ -340,3 +342,31 @@ fn error_block(p: &mut Parser<'_>, message: &str) {
     p.eat(T!['}']);
     m.complete(p, ERROR);
 }
+
+/// The `parser` passed this is required to at least consume one token if it returns `true`.
+/// If the `parser` returns false, parsing will stop.
+fn delimited(
+    p: &mut Parser<'_>,
+    bra: SyntaxKind,
+    ket: SyntaxKind,
+    delim: SyntaxKind,
+    first_set: TokenSet,
+    mut parser: impl FnMut(&mut Parser<'_>) -> bool,
+) {
+    p.bump(bra);
+    while !p.at(ket) && !p.at(EOF) {
+        if !parser(p) {
+            break;
+        }
+        if !p.at(delim) {
+            if p.at_ts(first_set) {
+                p.error(format!("expected {:?}", delim));
+            } else {
+                break;
+            }
+        } else {
+            p.bump(delim);
+        }
+    }
+    p.expect(ket);
+}

crates/parser/src/grammar/attributes.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+pub(super) const ATTRIBUTE_FIRST: TokenSet = TokenSet::new(&[T![#]]);
+
 pub(super) fn inner_attrs(p: &mut Parser<'_>) {
     while p.at(T![#]) && p.nth(1) == T![!] {
         attr(p, true);

crates/parser/src/grammar/expressions.rs

@@ -1,5 +1,7 @@
 mod atom;
 
+use crate::grammar::attributes::ATTRIBUTE_FIRST;
+
 use super::*;
 
 pub(crate) use self::atom::{block_expr, match_arm_list};
@@ -572,27 +574,11 @@ fn cast_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
 fn arg_list(p: &mut Parser<'_>) {
     assert!(p.at(T!['(']));
     let m = p.start();
-    p.bump(T!['(']);
+    // test arg_with_attr
-    while !p.at(T![')']) && !p.at(EOF) {
+    // fn main() {
-        // test arg_with_attr
+    //     foo(#[attr] 92)
-        // fn main() {
+    // }
-        //     foo(#[attr] 92)
+    delimited(p, T!['('], T![')'], T![,], EXPR_FIRST.union(ATTRIBUTE_FIRST), expr);
-        // }
-        if !expr(p) {
-            break;
-        }
-        if !p.at(T![,]) {
-            if p.at_ts(EXPR_FIRST) {
-                p.error("expected `,`");
-                continue;
-            } else {
-                break;
-            }
-        } else {
-            p.bump(T![,]);
-        }
-    }
-    p.eat(T![')']);
     m.complete(p, ARG_LIST);
 }
 

crates/parser/src/grammar/expressions/atom.rs

@@ -118,7 +118,7 @@ pub(super) fn atom_expr(
                     // fn main() {
                     //     'loop: impl
                     // }
-                    p.error("expected a loop");
+                    p.error("expected a loop or block");
                     m.complete(p, ERROR);
                     return None;
                 }

crates/parser/src/grammar/generic_args.rs

@@ -5,27 +5,35 @@ pub(super) fn opt_generic_arg_list(p: &mut Parser<'_>, colon_colon_required: boo
     if p.at(T![::]) && p.nth(2) == T![<] {
         m = p.start();
         p.bump(T![::]);
-        p.bump(T![<]);
     } else if !colon_colon_required && p.at(T![<]) && p.nth(1) != T![=] {
         m = p.start();
-        p.bump(T![<]);
     } else {
         return;
     }
 
-    while !p.at(EOF) && !p.at(T![>]) {
+    delimited(p, T![<], T![>], T![,], GENERIC_ARG_FIRST, generic_arg);
-        generic_arg(p);
-        if !p.at(T![>]) && !p.expect(T![,]) {
-            break;
-        }
-    }
-    p.expect(T![>]);
     m.complete(p, GENERIC_ARG_LIST);
 }
 
+const GENERIC_ARG_FIRST: TokenSet = TokenSet::new(&[
+    LIFETIME_IDENT,
+    IDENT,
+    T!['{'],
+    T![true],
+    T![false],
+    T![-],
+    INT_NUMBER,
+    FLOAT_NUMBER,
+    CHAR,
+    BYTE,
+    STRING,
+    BYTE_STRING,
+])
+.union(types::TYPE_FIRST);
+
 // test generic_arg
 // type T = S<i32>;
-fn generic_arg(p: &mut Parser<'_>) {
+fn generic_arg(p: &mut Parser<'_>) -> bool {
     match p.current() {
         LIFETIME_IDENT => lifetime_arg(p),
         T!['{'] | T![true] | T![false] | T![-] => const_arg(p),
@@ -68,8 +76,10 @@ fn generic_arg(p: &mut Parser<'_>) {
                 }
             }
         }
-        _ => type_arg(p),
+        _ if p.at_ts(types::TYPE_FIRST) => type_arg(p),
+        _ => return false,
     }
+    true
 }
 
 // test lifetime_arg

crates/parser/src/grammar/generic_params.rs

@@ -1,3 +1,5 @@
+use crate::grammar::attributes::ATTRIBUTE_FIRST;
+
 use super::*;
 
 pub(super) fn opt_generic_param_list(p: &mut Parser<'_>) {
@@ -11,32 +13,31 @@ pub(super) fn opt_generic_param_list(p: &mut Parser<'_>) {
 fn generic_param_list(p: &mut Parser<'_>) {
     assert!(p.at(T![<]));
     let m = p.start();
-    p.bump(T![<]);
+    delimited(p, T![<], T![>], T![,], GENERIC_PARAM_FIRST.union(ATTRIBUTE_FIRST), |p| {
+        // test generic_param_attribute
+        // fn foo<#[lt_attr] 'a, #[t_attr] T>() {}
+        let m = p.start();
+        attributes::outer_attrs(p);
+        generic_param(p, m)
+    });
 
-    while !p.at(EOF) && !p.at(T![>]) {
-        generic_param(p);
-        if !p.at(T![>]) && !p.expect(T![,]) {
-            break;
-        }
-    }
-    p.expect(T![>]);
     m.complete(p, GENERIC_PARAM_LIST);
 }
 
-fn generic_param(p: &mut Parser<'_>) {
+const GENERIC_PARAM_FIRST: TokenSet = TokenSet::new(&[IDENT, LIFETIME_IDENT, T![const]]);
-    let m = p.start();
+
-    // test generic_param_attribute
+fn generic_param(p: &mut Parser<'_>, m: Marker) -> bool {
-    // fn foo<#[lt_attr] 'a, #[t_attr] T>() {}
-    attributes::outer_attrs(p);
     match p.current() {
         LIFETIME_IDENT => lifetime_param(p, m),
         IDENT => type_param(p, m),
         T![const] => const_param(p, m),
         _ => {
             m.abandon(p);
-            p.err_and_bump("expected type parameter");
+            p.err_and_bump("expected generic parameter");
+            return false;
         }
     }
+    true
 }
 
 // test lifetime_param

crates/parser/src/grammar/items/adt.rs

@@ -1,3 +1,5 @@
+use crate::grammar::attributes::ATTRIBUTE_FIRST;
+
 use super::*;
 
 // test struct_item
@@ -141,28 +143,31 @@ pub(crate) fn record_field_list(p: &mut Parser<'_>) {
     }
 }
 
+const TUPLE_FIELD_FIRST: TokenSet =
+    types::TYPE_FIRST.union(ATTRIBUTE_FIRST).union(VISIBILITY_FIRST);
+
 fn tuple_field_list(p: &mut Parser<'_>) {
     assert!(p.at(T!['(']));
     let m = p.start();
-    p.bump(T!['(']);
+    delimited(p, T!['('], T![')'], T![,], TUPLE_FIELD_FIRST, |p| {
-    while !p.at(T![')']) && !p.at(EOF) {
         let m = p.start();
         // test tuple_field_attrs
         // struct S (#[attr] f32);
         attributes::outer_attrs(p);
-        opt_visibility(p, true);
+        let has_vis = opt_visibility(p, true);
         if !p.at_ts(types::TYPE_FIRST) {
             p.error("expected a type");
-            m.complete(p, ERROR);
+            if has_vis {
-            break;
+                m.complete(p, ERROR);
+            } else {
+                m.abandon(p);
+            }
+            return false;
         }
         types::type_(p);
         m.complete(p, TUPLE_FIELD);
+        true
+    });
 
-        if !p.at(T![')']) {
-            p.expect(T![,]);
-        }
-    }
-    p.expect(T![')']);
     m.complete(p, TUPLE_FIELD_LIST);
 }

crates/parser/src/grammar/params.rs

@@ -1,3 +1,5 @@
+use crate::grammar::attributes::ATTRIBUTE_FIRST;
+
 use super::*;
 
 // test param_list
@@ -66,14 +68,20 @@ fn list_(p: &mut Parser<'_>, flavor: Flavor) {
             }
         };
 
-        if !p.at_ts(PARAM_FIRST) {
+        if !p.at_ts(PARAM_FIRST.union(ATTRIBUTE_FIRST)) {
             p.error("expected value parameter");
             m.abandon(p);
             break;
         }
         param(p, m, flavor);
-        if !p.at(ket) {
+        if !p.at(T![,]) {
-            p.expect(T![,]);
+            if p.at_ts(PARAM_FIRST.union(ATTRIBUTE_FIRST)) {
+                p.error("expected `,`");
+            } else {
+                break;
+            }
+        } else {
+            p.bump(T![,]);
         }
     }
 

crates/parser/test_data/parser/err/0009_broken_struct_type_parameter.rast

@@ -44,8 +44,7 @@ SOURCE_FILE
       IDENT "T"
     SEMICOLON ";"
   WHITESPACE "\n"
-error 9: expected type parameter
+error 9: expected generic parameter
-error 11: expected COMMA
 error 11: expected R_ANGLE
 error 11: expected `;`, `{`, or `(`
 error 12: expected an item

crates/parser/test_data/parser/err/0013_invalid_type.rast

@@ -43,17 +43,14 @@ SOURCE_FILE
                                     IDENT "Box"
                                   GENERIC_ARG_LIST
                                     L_ANGLE "<"
-                                    TYPE_ARG
+  ERROR
-                                      ERROR
+    AT "@"
-                                        AT "@"
+  WHITESPACE " "
-      WHITESPACE " "
+  MACRO_CALL
-      TUPLE_FIELD
+    PATH
-        PATH_TYPE
+      PATH_SEGMENT
-          PATH
+        NAME_REF
-            PATH_SEGMENT
+          IDENT "Any"
-              NAME_REF
-                IDENT "Any"
-      ERROR
   ERROR
     R_ANGLE ">"
   ERROR
@@ -69,17 +66,14 @@ SOURCE_FILE
   ERROR
     SEMICOLON ";"
   WHITESPACE "\n\n"
-error 67: expected type
+error 67: expected R_ANGLE
-error 68: expected COMMA
+error 67: expected R_ANGLE
-error 68: expected R_ANGLE
+error 67: expected R_ANGLE
-error 68: expected COMMA
+error 67: expected R_PAREN
-error 68: expected R_ANGLE
+error 67: expected SEMICOLON
-error 68: expected COMMA
+error 67: expected an item
-error 68: expected R_ANGLE
+error 72: expected BANG
-error 68: expected COMMA
+error 72: expected `{`, `[`, `(`
-error 72: expected COMMA
-error 72: expected a type
-error 72: expected R_PAREN
 error 72: expected SEMICOLON
 error 72: expected an item
 error 73: expected an item

crates/parser/test_data/parser/err/0022_bad_exprs.rast

@@ -151,6 +151,7 @@ error 26: expected `;`, `{`, or `(`
 error 30: expected pattern
 error 31: expected SEMICOLON
 error 53: expected expression
+error 54: expected R_PAREN
 error 54: expected SEMICOLON
 error 54: expected expression, item or let statement
 error 60: expected type
@@ -160,6 +161,7 @@ error 65: expected pattern
 error 65: expected SEMICOLON
 error 65: expected expression, item or let statement
 error 92: expected expression
+error 93: expected R_PAREN
 error 93: expected SEMICOLON
 error 93: expected expression, item or let statement
 error 95: expected expression, item or let statement

crates/parser/test_data/parser/err/0024_many_type_parens.rast

@@ -168,12 +168,12 @@ SOURCE_FILE
                       L_PAREN "("
                       ERROR
                         QUESTION "?"
-        EXPR_STMT
+                  TYPE_ARG
-          PATH_EXPR
+                    PATH_TYPE
-            PATH
+                      PATH
-              PATH_SEGMENT
+                        PATH_SEGMENT
-                NAME_REF
+                          NAME_REF
-                  IDENT "Sized"
+                            IDENT "Sized"
         ERROR
           R_PAREN ")"
         WHITESPACE " "
@@ -291,15 +291,13 @@ SOURCE_FILE
         WHITESPACE "\n"
         R_CURLY "}"
   WHITESPACE "\n"
-error 88: expected COMMA
 error 88: expected R_ANGLE
 error 121: expected SEMICOLON
 error 121: expected expression, item or let statement
 error 140: expected type
 error 141: expected R_PAREN
 error 141: expected COMMA
-error 141: expected R_ANGLE
+error 146: expected R_ANGLE
-error 141: expected SEMICOLON
 error 146: expected SEMICOLON
 error 146: expected expression, item or let statement
 error 148: expected expression, item or let statement
@@ -309,7 +307,6 @@ error 165: expected expression
 error 168: expected expression
 error 179: expected expression
 error 180: expected SEMICOLON
-error 215: expected COMMA
 error 215: expected R_ANGLE
 error 235: expected SEMICOLON
 error 235: expected expression, item or let statement

crates/parser/test_data/parser/err/0025_nope.rast

@@ -156,8 +156,7 @@ SOURCE_FILE
                             PATH_SEGMENT
                               NAME_REF
                                 IDENT "i32"
-                      WHITESPACE " "
+                  WHITESPACE " "
-                      ERROR
                   ERROR
                     L_CURLY "{"
                     R_CURLY "}"
@@ -199,10 +198,8 @@ error 95: expected type
 error 95: expected COMMA
 error 96: expected field
 error 98: expected field declaration
+error 371: expected R_PAREN
 error 371: expected COMMA
-error 372: expected a type
-error 372: expected R_PAREN
-error 372: expected COMMA
 error 372: expected enum variant
 error 374: expected enum variant
 error 494: expected pattern

crates/parser/test_data/parser/err/0042_weird_blocks.rast

@@ -72,4 +72,4 @@ SOURCE_FILE
 error 24: expected existential, fn, trait or impl
 error 41: expected existential, fn, trait or impl
 error 56: expected a block
-error 75: expected a loop
+error 75: expected a loop or block

crates/parser/test_data/parser/err/0048_double_fish.rast

@@ -12,7 +12,7 @@ SOURCE_FILE
       STMT_LIST
         L_CURLY "{"
         WHITESPACE "\n    "
-        EXPR_STMT
+        BIN_EXPR
           PATH_EXPR
             PATH
               PATH_SEGMENT
@@ -41,13 +41,14 @@ SOURCE_FILE
                         COLON2 "::"
                         ERROR
                           L_ANGLE "<"
-        BIN_EXPR
+                  TYPE_ARG
-          PATH_EXPR
+                    PATH_TYPE
-            PATH
+                      PATH
-              PATH_SEGMENT
+                        PATH_SEGMENT
-                NAME_REF
+                          NAME_REF
-                  IDENT "nope"
+                            IDENT "nope"
-          SHR ">>"
+                  R_ANGLE ">"
+          R_ANGLE ">"
           ERROR
             SEMICOLON ";"
         WHITESPACE "\n"
@@ -114,8 +115,6 @@ SOURCE_FILE
   WHITESPACE "\n"
 error 30: expected identifier
 error 31: expected COMMA
-error 31: expected R_ANGLE
-error 31: expected SEMICOLON
 error 37: expected expression
 error 75: expected identifier
 error 76: expected SEMICOLON

crates/parser/test_data/parser/inline/err/0002_misplaced_label_err.rast

@@ -23,6 +23,6 @@ SOURCE_FILE
         WHITESPACE "\n"
         R_CURLY "}"
   WHITESPACE "\n"
-error 22: expected a loop
+error 22: expected a loop or block
 error 27: expected type
 error 27: expected `{`

crates/parser/test_data/parser/inline/err/0015_arg_list_recovery.rast

@@ -72,6 +72,6 @@ SOURCE_FILE
         R_CURLY "}"
   WHITESPACE "\n"
 error 25: expected identifier
-error 39: expected `,`
+error 39: expected COMMA
 error 39: expected expression
 error 55: expected expression

crates/parser/test_data/parser/inline/err/0015_missing_fn_param_type.rast

@@ -49,5 +49,5 @@ SOURCE_FILE
         R_CURLY "}"
   WHITESPACE "\n"
 error 6: missing type for function parameter
-error 6: expected COMMA
+error 6: expected `,`
 error 16: missing type for function parameter