Teach parser_t how to parse an argument list that contains newlines, for

complete -a support. Fixes #1369

parse_constants.h

@@ -43,6 +43,10 @@ enum parse_token_type_t
     symbol_argument_or_redirection,
 
     symbol_argument_list,
+    
+    // "freestanding" argument lists are parsed from the argument list supplied to 'complete -a'
+    // They are not generated by parse trees rooted in symbol_job_list
+    symbol_freestanding_argument_list,
 
     symbol_argument,
     symbol_redirection,
@@ -50,7 +54,7 @@ enum parse_token_type_t
     symbol_optional_background,
 
     symbol_end_command,
-
+    
     // Terminal types
     parse_token_type_string,
     parse_token_type_pipe,

parse_productions.cpp

@@ -276,6 +276,26 @@ RESOLVE(argument_list)
     }
 }
 
+PRODUCTIONS(freestanding_argument_list) =
+{
+    {},
+    {symbol_argument, symbol_freestanding_argument_list},
+    {parse_token_type_end, symbol_freestanding_argument_list},
+};
+RESOLVE(freestanding_argument_list)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_string:
+            return 1;
+        case parse_token_type_end:
+            return 2;
+        default:
+            return 0;
+    }
+}
+
+
 PRODUCTIONS(block_statement) =
 {
     {symbol_block_header, parse_token_type_end, symbol_job_list, symbol_end_command, symbol_arguments_or_redirections_list}
@@ -485,6 +505,7 @@ const production_t *parse_productions::production_for_token(parse_token_type_t n
             TEST(case_item_list)
             TEST(case_item)
             TEST(argument_list)
+            TEST(freestanding_argument_list)
             TEST(block_header)
             TEST(for_header)
             TEST(while_header)

parse_tree.cpp

@@ -186,6 +186,8 @@ wcstring token_type_description(parse_token_type_t type)
 
         case symbol_argument_list:
             return L"argument_list";
+        case symbol_freestanding_argument_list:
+            return L"freestanding_argument_list";
 
         case symbol_boolean_statement:
             return L"boolean_statement";

parse_tree.h

@@ -254,7 +254,12 @@ bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags, parse
     optional_background = <empty> | <TOK_BACKGROUND>
 
     end_command = END
-
+ 
+ # A freestanding_argument_list is equivalent to a normal argument list, except it may contain TOK_END (newlines, and even semicolons, for historical reasons:
+ 
+     freestanding_argument_list = <empty> |
+                                  argument freestanding_argument_list |
+                                  <TOK_END> freestanding_argument_list
 */
 
 #endif

parser.cpp

@@ -493,7 +493,7 @@ void parser_t::expand_argument_list(const wcstring &arg_list_src, std::vector<co
 
     /* Parse the string as an argument list */
     parse_node_tree_t tree;
-    if (! parse_tree_from_string(arg_list_src, parse_flag_none, &tree, NULL /* errors */, symbol_argument_list))
+    if (! parse_tree_from_string(arg_list_src, parse_flag_none, &tree, NULL /* errors */, symbol_freestanding_argument_list))
     {
         /* Failed to parse. Here we expect to have reported any errors in test_args */
         return;
@@ -502,7 +502,7 @@ void parser_t::expand_argument_list(const wcstring &arg_list_src, std::vector<co
     /* Get the root argument list */
     assert(! tree.empty());
     const parse_node_t *arg_list = &tree.at(0);
-    assert(arg_list->type == symbol_argument_list);
+    assert(arg_list->type == symbol_freestanding_argument_list);
 
     /* Extract arguments from it */
     while (arg_list != NULL)
@@ -968,18 +968,18 @@ bool parser_t::detect_errors_in_argument_list(const wcstring &arg_list_src, wcst
 
     /* Parse the string as an argument list */
     parse_node_tree_t tree;
-    if (! parse_tree_from_string(arg_list_src, parse_flag_none, &tree, &errors, symbol_argument_list))
+    if (! parse_tree_from_string(arg_list_src, parse_flag_none, &tree, &errors, symbol_freestanding_argument_list))
     {
         /* Failed to parse. */
         errored = true;
     }
-
+    
     if (! errored)
     {
         /* Get the root argument list */
         assert(! tree.empty());
         const parse_node_t *arg_list = &tree.at(0);
-        assert(arg_list->type == symbol_argument_list);
+        assert(arg_list->type == symbol_freestanding_argument_list);
 
         /* Extract arguments from it */
         while (arg_list != NULL && ! errored)