Parser limps into existence

Makefile.in

@@ -95,7 +95,7 @@ FISH_OBJS := function.o builtin.o complete.o env.o exec.o expand.o		\
 	env_universal.o env_universal_common.o input_common.o event.o		\
 	signal.o io.o parse_util.o common.o screen.o path.o autoload.o		\
 	parser_keywords.o iothread.o color.o postfork.o	\
-	builtin_test.o
+	builtin_test.o expression.o
 
 FISH_INDENT_OBJS := fish_indent.o print_help.o common.o	\
 parser_keywords.o wutil.o tokenizer.o

configure.ac

@@ -92,7 +92,7 @@ fi
 # So ensure this happens before we modify CXXFLAGS below
 #
 
-AC_PROG_CXX([g++ c++])
+AC_PROG_CXX([clang++ g++ c++])
 AC_PROG_CPP
 AC_PROG_INSTALL
 

expression.cpp

@@ -1,7 +1,8 @@
 #include "expression.h"
-#include <stack>
+#include "tokenizer.h"
 #include <vector>
 
+
 /* Fish grammar:
 
 # A statement_list is a list of statements, separated by semicolons or newlines
@@ -41,11 +42,12 @@
  
 */
 
+typedef size_t node_offset_t;
+
 #define PARSE_ASSERT(a) assert(a)
 
 #define PARSER_DIE() assert(0)
 
-#if 1
 class parse_command_t;
 
 enum parse_token_type_t
@@ -67,7 +69,6 @@ enum parse_token_type_t
     symbol_plain_statement,
     symbol_arguments_or_redirections_list,
     symbol_argument_or_redirection,
-    
 
     // Terminal types
     parse_token_type_string,
@@ -76,8 +77,42 @@ enum parse_token_type_t
     parse_token_background,
     parse_token_type_end,
     parse_token_type_terminate,
+    
+    FIRST_PARSE_TOKEN_TYPE = parse_token_type_string
 };
 
+static wcstring token_type_description(parse_token_type_t type)
+{
+    switch (type)
+    {
+        case token_type_invalid: return L"invalid";
+        
+        case symbol_statement_list: return L"statement_list";
+        case symbol_statement: return L"statement";
+        case symbol_block_statement: return L"block_statement";
+        case symbol_block_header: return L"block_header";
+        case symbol_if_header: return L"if_header";
+        case symbol_for_header: return L"for_header";
+        case symbol_while_header: return L"while_header";
+        case symbol_begin_header: return L"begin_header";
+        case symbol_function_header: return L"function_header";
+        case symbol_boolean_statement: return L"boolean_statement";
+        case symbol_decorated_statement: return L"decorated_statement";
+        case symbol_plain_statement: return L"plain_statement";
+        case symbol_arguments_or_redirections_list: return L"arguments_or_redirections_list";
+        case symbol_argument_or_redirection: return L"argument_or_redirection";
+        
+        case parse_token_type_string: return L"token_string";
+        case parse_token_type_pipe: return L"token_pipe";
+        case parse_token_type_redirection: return L"token_redirection";
+        case parse_token_background: return L"token_background";
+        case parse_token_type_end: return L"token_end";
+        case parse_token_type_terminate: return L"token_terminate";
+        
+        default: return format_string(L"Unknown token type %ld", static_cast<long>(type));
+    }
+}
+
 enum parse_keyword_t
 {
     parse_keyword_none,
@@ -100,11 +135,11 @@ enum parse_keyword_t
 
 struct parse_token_t
 {
-    enum parse_token_type_t type; // The type of the token as represnted by the parser
+    enum parse_token_type_t type; // The type of the token as represented by the parser
     enum token_type tokenizer_type; // The type of the token as represented by the tokenizer
     enum parse_keyword_t keyword; // Any keyword represented by this parser
     size_t source_start;
-    size_t source_end;
+    size_t source_length;
 };
 
 // Convert from tokenizer_t's token type to our token
@@ -138,253 +173,114 @@ static parse_token_t parse_token_from_tokenizer_token(enum token_type tokenizer_
     return result;
 }
 
-/** Root of a parse tree */
-class parse_statement_list_t;
-class parse_tree_t
-{
-    friend class parse_ll_t;
-    
-    parse_statement_list_t *root;
-};
-
 /** Base class for nodes of a parse tree */
-class parse_node_base_t
+class parse_node_t
 {
-    /* Type of the node */
+    public:
-    const enum parse_token_type_t type;
+        
-    
     /* Start in the source code */
-    const unsigned int source_start;
+    size_t source_start;
     
     /* Length of our range in the source code */
-    const unsigned int source_length;
+    size_t source_length;
+
+    /* Children */
+    node_offset_t child_start;
+    node_offset_t child_count;
     
-    protected:
+    /* Type-dependent data */
-    /* Index of the production used */
+    uint32_t tag;
-    unsigned char branch;
     
-    public:
+    /* Type of the node */
-    parse_node_base_t(parse_token_type_t ty) : type(ty), source_start(0), source_length(0)
+    enum parse_token_type_t type;
+
+    
+    /* Description */
+    wcstring describe(void) const
     {
+        wcstring result = token_type_description(type);
+        return result;
     }
     
-    virtual ~parse_node_base_t()
+    /* Constructor */
+    explicit parse_node_t(parse_token_type_t ty) : type(ty), source_start(0), source_length(0), child_start(0), child_count(0), tag(0)
     {
     }
 };
 
-class parse_node_t : public parse_node_base_t
+static void dump_tree_recursive(const std::vector<parse_node_t> &nodes, const wcstring &src, size_t start, size_t indent, wcstring *result, size_t *line)
 {
-    public:
+    assert(start < nodes.size());
-    parse_node_t *p1;
+    const parse_node_t &node = nodes.at(start);
-    parse_node_t *p2;
-    uint32_t c1;
     
-    parse_node_t(parse_token_type_t ty) : parse_node_base_t(ty), p1(NULL), p2(NULL), c1(0)
+    append_format(*result, L"%2lu   ", *line);
+    result->append(indent, L' ');;
+    result->append(node.describe());
+    if (node.child_count > 0)
     {
+        append_format(*result, L" <%lu children>", node.child_count);
     }
-};
+    if (node.type == parse_token_type_string)
-
-class parse_statement_t;
-class parse_statement_list_t : public parse_node_base_t
-{
-    public:
-    parse_statement_t *statement;
-    parse_statement_list_t *next;
-    parse_statement_list_t() : parse_node_base_t(symbol_statement_list), statement(NULL), next(NULL)
     {
+        result->append(L": \"");
+        result->append(src, node.source_start, node.source_length);
+        result->append(L"\"");
     }
-};
+    result->push_back(L'\n');
-
+    ++*line;
-class parse_statement_t : public parse_node_base_t
+    for (size_t child_idx = node.child_start; child_idx < node.child_start + node.child_count; child_idx++)
-{
-    // abstract class
-    public:
-    parse_statement_t(parse_token_type_t ty) : parse_node_base_t(ty)
     {
+        dump_tree_recursive(nodes, src, child_idx, indent + 2, result, line);
     }
-};
+}
 
-class parse_abstract_statement_t : public parse_statement_t
+static wcstring dump_tree(const std::vector<parse_node_t> &nodes, const wcstring &src)
 {
-    public:
+    if (nodes.empty())
-    parse_statement_t *subject;
+        return L"(empty!)";
-    parse_abstract_statement_t() : parse_statement_t(symbol_statement), subject(NULL)
-    {
-    }
-};
-
-class parse_boolean_statement_t : public parse_statement_t
-{
-    public:
-    enum {
-        boolean_invalid,
-        boolean_and,
-        boolean_or,
-        boolean_not
-    } condition;
     
-    parse_boolean_statement_t() : parse_statement_t(symbol_boolean_statement), condition(boolean_invalid)
+    size_t line = 0;
-    {
+    wcstring result;
-#if 0
+    dump_tree_recursive(nodes, src, 0, 0, &result, &line);
-        switch (keyword)
+    return result;
-        {
+}
-            case parse_keyword_and:
-                condition = boolean_and;
-                break;
-            
-            case parse_keyword_or:
-                condition = boolean_or;
-                break;
-            
-            case parse_keyword_not:
-                condition = boolean_not;
-                break;
-                
-            default:
-                PARSE_ASSERT(0 && "Unknown keyword");
-                break;
-        }
-#endif
-    }
-};
-
-class parse_plain_statement_t;
-class parse_decorated_statement_t : public parse_statement_t
-{
-public:
-    enum {
-        decoration_none,
-        decoration_command,
-        decoration_builtin
-    } decoration;
-    
-    parse_plain_statement_t *subject;
-    
-    parse_decorated_statement_t() : parse_statement_t(symbol_decorated_statement), subject(NULL), decoration(decoration_none)
-    {
-    }
-
-};
-
-class parse_string_t;
-class parse_plain_statement_t : public parse_statement_t
-{
-    parse_string_t *command;
-    parse_arguments_or_redirection_list_t *arguments_or_redirections_list;
-    
-    public:
-    parse_plain_statement_t() : parse_statement_t(symbol_plain_statement)
-    {
-    }
-};
-
-class parse_block_statement_t : public parse_statement_t
-{
-    // abstract class
-    parse_block_statement_t(parse_tree_t *t, parse_token_type_t ty) : parse_statement_t(ty)
-    {
-    }
-};
-
-class parse_string_t : public parse_node_base_t
-{
-};
-
-class parse_arguments_or_redirection_list_t : public parse_node_base_t
-{
-};
 
 
 struct parse_stack_element_t
 {
     enum parse_token_type_t type;
     enum parse_keyword_t keyword;
-    parse_node_base_t *node;
+    node_offset_t node_idx;
     
-    private:
+    parse_stack_element_t(parse_token_type_t t) : type(t), keyword(parse_keyword_none), node_idx(-1)
-    void allocate_node(void)
     {
-        assert(node == NULL);
-        switch (type)
-        {
-            // Set up our node
-            case symbol_statement_list:
-                node = new parse_statement_list_t();
-                break;
-                
-            case symbol_statement:
-                node = new parse_abstract_statement_t();
-                break;
-                
-            case symbol_block_statement:
-            case symbol_block_header:
-            case symbol_if_header:
-            case symbol_for_header:
-            case symbol_while_header:
-            case symbol_begin_header:
-            case symbol_function_header:
-                break;
-            
-            case symbol_boolean_statement:
-                node = new parse_boolean_statement_t();
-                break;
-                
-            case symbol_decorated_statement:
-                node = new parse_decorated_statement_t();
-                break;
-                
-            case symbol_plain_statement:
-                node = new parse_plain_statement_t();
-                break;
-                
-            case symbol_arguments_or_redirections_list:
-            case symbol_argument_or_redirection:
-            
-            default:
-            ;
-                // nothing
-        }
-    }
-
-    
-    public:
-    
-    // Construct a token type, with no keyword
-    parse_stack_element_t(enum parse_token_type_t t) : type(t), keyword(parse_keyword_none)
-    {
-        allocate_node();
     }
     
-    // Construct a string type from a keyword
+    parse_stack_element_t(parse_keyword_t k) : type(parse_token_type_string), keyword(k), node_idx(-1)
-    parse_stack_element_t(enum parse_keyword_t k) : type(parse_token_type_string), keyword(k), node(NULL)
     {
-        allocate_node();
     }
 };
 
-template<typename T>
-static T* cast_node(parse_node_base_t *node)
-{
-    return static_cast<T*>(node);
-}
-
 class parse_ll_t
 {
     friend class parse_t;
     
     std::vector<parse_stack_element_t> symbol_stack; // LL parser stack
-    parse_tree_t *tree; //tree we are constructing
+    std::vector<parse_node_t> nodes;
+    bool errored;
     
     // Constructor
-    parse_ll_t()
+    parse_ll_t() : errored(false)
     {
-        this->tree = new parse_tree_t();
+        // initial node
-
+        parse_stack_element_t elem = symbol_statement_list;
-        symbol_stack.push_back(symbol_statement_list); // goal token
+        elem.node_idx = 0;
-        tree->root = stack_get_node_cast<parse_statement_list_t>(0);
+        symbol_stack.push_back(elem); // goal token
+        nodes.push_back(parse_node_t(symbol_statement_list));
     }
     
+    bool top_node_match_token(parse_token_t token);
+    
     // implementation of certain parser constructions
     void accept_token(parse_token_t token);
     void accept_token_statement_list(parse_token_t token);
@@ -395,34 +291,63 @@ class parse_ll_t
     void accept_token_plain_statement(parse_token_t token);
     void accept_token_arguments_or_redirections_list(parse_token_t token);
     void accept_token_argument_or_redirection(parse_token_t token);
+    bool accept_token_string(parse_token_t token);
     
     void token_unhandled(parse_token_t token, const char *function);
     
     void parse_error(const wchar_t *expected, parse_token_t token);
     
+    // Get the node corresponding to the top element of the stack
+    parse_node_t &node_for_top_symbol()
+    {
+        PARSE_ASSERT(! symbol_stack.empty());
+        const parse_stack_element_t &top_symbol = symbol_stack.back();
+        PARSE_ASSERT(top_symbol.node_idx != -1);
+        PARSE_ASSERT(top_symbol.node_idx < nodes.size());
+        return nodes.at(top_symbol.node_idx);
+    }
+    
     parse_token_type_t stack_top_type() const
     {
         return symbol_stack.back().type;
     }
     
-    template<typename T>
+    void top_node_set_tag(uint32_t tag)
-    T* stack_get_node_cast(unsigned int idx)
     {
-        assert(idx < symbol_stack.size());
+        this->node_for_top_symbol().tag = tag;
-        parse_node_base_t *base_node = symbol_stack.at(symbol_stack.size() - idx - 1).node;
+    }
-        return static_cast<T *>(base_node);
+    
+    inline void add_child_to_node(size_t parent_node_idx, parse_stack_element_t *tok)
+    {
+        PARSE_ASSERT(tok->type != token_type_invalid);
+        tok->node_idx = nodes.size();
+        nodes.push_back(parse_node_t(tok->type));
+        nodes.at(parent_node_idx).child_count += 1;
+    }
 
-    }
     
-    parse_node_base_t *stack_get_node(unsigned int idx) const
+    // Pop from the top of the symbol stack, then push, updating node counts. Note that these are pushed in reverse order, so the first argument will be on the top of the stack.
-    {
-        assert(idx < symbol_stack.size());
-        return symbol_stack.at(symbol_stack.size() - idx - 1).node;
-    }
-    
-    // Pop from the top of the symbol stack, then push. Note that these are pushed in reverse order, so the first argument will be on the top of the stack
     inline void symbol_stack_pop_push(parse_stack_element_t tok1 = token_type_invalid, parse_stack_element_t tok2 = token_type_invalid, parse_stack_element_t tok3 = token_type_invalid, parse_stack_element_t tok4 = token_type_invalid, parse_stack_element_t tok5 = token_type_invalid)
     {
+        // Get the node for the top symbol and tell it about its children
+        size_t node_idx = symbol_stack.back().node_idx;
+        parse_node_t &node = nodes.at(node_idx);
+        
+        // Should have no children yet
+        PARSE_ASSERT(node.child_count == 0);
+        
+        // Tell the node where its children start
+        node.child_start = nodes.size();
+        
+        // Add nodes for the children
+        // Confusingly, we want our nodes to be in forwards order (last token last, so dumps look nice), but the symbols should be reverse order (last token first, so it's lowest on the stack)
+        if (tok1.type != token_type_invalid) add_child_to_node(node_idx, &tok1);
+        if (tok2.type != token_type_invalid) add_child_to_node(node_idx, &tok2);
+        if (tok3.type != token_type_invalid) add_child_to_node(node_idx, &tok3);
+        if (tok4.type != token_type_invalid) add_child_to_node(node_idx, &tok4);
+        if (tok5.type != token_type_invalid) add_child_to_node(node_idx, &tok5);
+        
+        // The above set the node_idx. Now replace the top of the stack.
         symbol_stack.pop_back();
         if (tok5.type != token_type_invalid) symbol_stack.push_back(tok5);
         if (tok4.type != token_type_invalid) symbol_stack.push_back(tok4);
@@ -446,7 +371,6 @@ void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token)
 void parse_ll_t::accept_token_statement_list(parse_token_t token)
 {
     PARSE_ASSERT(stack_top_type() == symbol_statement_list);
-    parse_statement_list_t *list = stack_get_node_cast<parse_statement_list_t>(0);
     switch (token.type)
     {
         case parse_token_type_string:
@@ -455,8 +379,6 @@ void parse_ll_t::accept_token_statement_list(parse_token_t token)
         case parse_token_background:
         case parse_token_type_end:
             symbol_stack_pop_push(symbol_statement, symbol_statement_list);
-            list->next = stack_get_node_cast<parse_statement_list_t>(0);
-            list->statement = stack_get_node_cast<parse_statement_t>(1);
             break;
             
         case parse_token_type_terminate:
@@ -473,7 +395,6 @@ void parse_ll_t::accept_token_statement_list(parse_token_t token)
 void parse_ll_t::accept_token_statement(parse_token_t token)
 {
     PARSE_ASSERT(stack_top_type() == symbol_statement);
-    parse_abstract_statement_t *statement = stack_get_node_cast<parse_abstract_statement_t>(0);
     switch (token.type)
     {
         case parse_token_type_string:
@@ -483,7 +404,6 @@ void parse_ll_t::accept_token_statement(parse_token_t token)
                 case parse_keyword_or:
                 case parse_keyword_not:
                     symbol_stack_pop_push(symbol_boolean_statement);
-                    statement->subject = stack_get_node_cast<parse_boolean_statement_t>(0);
                     break;
 
                 case parse_keyword_if:
@@ -506,7 +426,6 @@ void parse_ll_t::accept_token_statement(parse_token_t token)
                 case parse_keyword_command:
                 case parse_keyword_builtin:
                     symbol_stack_pop_push(symbol_decorated_statement);
-                    statement->subject = stack_get_node_cast<parse_decorated_statement_t>(0);
                     break;
                     
             }
@@ -575,23 +494,16 @@ void parse_ll_t::accept_token_block_header(parse_token_t token)
 void parse_ll_t::accept_token_boolean_statement(parse_token_t token)
 {
     PARSE_ASSERT(stack_top_type() == symbol_boolean_statement);
-    parse_boolean_statement_t *statement = stack_get_node_cast<parse_boolean_statement_t>(0);
     switch (token.type)
     {
         case parse_token_type_string:
             switch (token.keyword)
             {
                 case parse_keyword_and:
-                    statement->condition = parse_boolean_statement_t::boolean_and;
-                    symbol_stack_pop_push(parse_keyword_and, symbol_statement);
-                    break;
                 case parse_keyword_or:
-                    statement->condition = parse_boolean_statement_t::boolean_or;
-                    symbol_stack_pop_push(parse_keyword_or, symbol_statement);
-                    break;
                 case parse_keyword_not:
-                    statement->condition = parse_boolean_statement_t::boolean_not;
+                    top_node_set_tag(token.keyword);
-                    symbol_stack_pop_push(parse_keyword_not, symbol_statement);
+                    symbol_stack_pop_push(token.keyword, symbol_statement);
                     break;
                     
                 default:
@@ -608,31 +520,27 @@ void parse_ll_t::accept_token_boolean_statement(parse_token_t token)
 void parse_ll_t::accept_token_decorated_statement(parse_token_t token)
 {
     PARSE_ASSERT(stack_top_type() == symbol_decorated_statement);
-    parse_decorated_statement_t *statement = stack_get_node_cast<parse_decorated_statement_t>(0);
     switch (token.type)
     {
         case parse_token_type_string:
             switch (token.keyword)
             {
                 case parse_keyword_command:
+                    top_node_set_tag(parse_keyword_command);
                     symbol_stack_pop_push(parse_keyword_command, symbol_plain_statement);
-                    statement->subject = stack_get_node_cast<parse_plain_statement_t>(0);
-                    statement->decoration = parse_decorated_statement_t::decoration_command;
-
                     break;
                     
                 case parse_keyword_builtin:
+                    top_node_set_tag(parse_keyword_builtin);
                     symbol_stack_pop_push(parse_keyword_builtin, symbol_plain_statement);
-                    statement->subject = stack_get_node_cast<parse_plain_statement_t>(0);
-                    statement->decoration = parse_decorated_statement_t::decoration_builtin;
                     break;
                     
                 default:
+                    top_node_set_tag(parse_keyword_none);
                     symbol_stack_pop_push(symbol_plain_statement);
-                    statement->subject = stack_get_node_cast<parse_plain_statement_t>(0);
-                    statement->decoration = parse_decorated_statement_t::decoration_none;
                     break;
             }
+            break;
             
         default:
             token_unhandled(token, __FUNCTION__);
@@ -642,10 +550,8 @@ void parse_ll_t::accept_token_decorated_statement(parse_token_t token)
 
 void parse_ll_t::accept_token_plain_statement(parse_token_t token)
 {
-    PARSE_ASSERT(stack_top_type() == symbol_decorated_statement);
+    PARSE_ASSERT(stack_top_type() == symbol_plain_statement);
-    parse_plain_statement_t *statement = stack_get_node_cast<parse_plain_statement_t>(0);
     symbol_stack_pop_push(parse_token_type_string, symbol_arguments_or_redirections_list, parse_token_type_end);
-    statement->
 }
 
 void parse_ll_t::accept_token_arguments_or_redirections_list(parse_token_t token)
@@ -671,12 +577,12 @@ void parse_ll_t::accept_token_argument_or_redirection(parse_token_t token)
     switch (token.type)
     {
         case parse_token_type_string:
-            symbol_stack_pop_push();
+            symbol_stack_pop_push(parse_token_type_string);
             // Got an argument
             break;
             
         case parse_token_type_redirection:
-            symbol_stack_pop_push();
+            symbol_stack_pop_push(parse_token_type_redirection);
             // Got a redirection
             break;
             
@@ -686,105 +592,198 @@ void parse_ll_t::accept_token_argument_or_redirection(parse_token_t token)
     }
 }
 
-void parse_ll_t::accept_token(parse_token_t token)
+bool parse_ll_t::accept_token_string(parse_token_t token)
 {
-    assert(! symbol_stack.empty());
+    PARSE_ASSERT(stack_top_type() == parse_token_type_string);
-    switch (stack_top_type())
+    bool result = false;
-    {
-        case symbol_statement_list:
-            accept_token_statement_list(token);
-            break;
-            
-        case symbol_statement:
-            accept_token_statement(token);
-            break;
-            
-        case symbol_block_statement:
-            symbol_stack_pop_push(symbol_block_header, symbol_statement_list, parse_keyword_end, symbol_arguments_or_redirections_list);
-            break;
-            
-        case symbol_block_header:
-            accept_token_block_header(token);
-            break;
-            
-        case symbol_if_header:
-            break;
-            
-        case symbol_for_header:
-            symbol_stack_pop_push(parse_keyword_for, parse_token_type_string, parse_keyword_in, symbol_arguments_or_redirections_list, parse_token_type_end);
-            break;
-            
-        case symbol_while_header:
-            symbol_stack_pop_push(parse_keyword_while, symbol_statement);
-            break;
-            
-        case symbol_begin_header:
-            symbol_stack_pop_push(parse_keyword_begin, parse_token_type_end);
-            break;
-            
-        case symbol_function_header:
-            symbol_stack_pop_push(parse_keyword_function, symbol_arguments_or_redirections_list, parse_token_type_end);
-            break;
-            
-        case symbol_boolean_statement:
-            accept_token_boolean_statement(token);
-            break;
-            
-        case symbol_decorated_statement:
-            accept_token_decorated_statement(token);
-            break;
-            
-        case symbol_plain_statement:
-            accept_token_plain_statement(token);
-            break;
-            
-        case symbol_arguments_or_redirections_list:
-            accept_token_arguments_or_redirections_list(token);
-            break;
-            
-        case symbol_argument_or_redirection:
-            accept_token_argument_or_redirection(token);
-            break;
-    }
-}
-#endif
-
-#if 0
-class parse_sr_t
-{
-    friend class parse_t;
-    
-    std::vector<parse_node_base_t *> node_stack;
-    void accept_token(parse_token_t token);
-    
-    void accept_token_string(parse_token_t token);
-};
-
-void parse_sr_t::accept_token_string(parse_token_t token)
-{
-    assert(token.type == parse_token_type_string);
-}
-
-void parse_sr_t::accept_token(parse_token_t token)
-{
-    // We are a SR parser. Our action depends on a combination of the top element(s) of our node stack and the token type.
-    // Switch on the token type to make progress
     switch (token.type)
     {
         case parse_token_type_string:
-            accept_token_string(token);
+            // Got our string
+            symbol_stack_pop_push();
+            result = true;
             break;
             
-        case parse_token_type_pipe:
+        default:
-        case parse_token_type_redirection:
+            token_unhandled(token, __FUNCTION__);
-        case parse_token_background:
+            break;
-        case parse_token_type_end:
-        case parse_token_type_terminate:
     }
+    return result;
 }
 
-#endif
+bool parse_ll_t::top_node_match_token(parse_token_t token)
+{
+    PARSE_ASSERT(! symbol_stack.empty());
+    PARSE_ASSERT(token.type >= FIRST_PARSE_TOKEN_TYPE);
+    bool result = false;
+    parse_stack_element_t &stack_top = symbol_stack.back();
+    if (stack_top.type == token.type)
+    {
+        // So far so good. See if we need a particular keyword.
+        if (stack_top.keyword == parse_keyword_none || stack_top.keyword == token.keyword)
+        {
+            // Success. Tell the node that it matched this token
+            parse_node_t &node = node_for_top_symbol();
+            node.source_start = token.source_start;
+            node.source_length = token.source_length;
+            
+            // We consumed this symbol
+            symbol_stack.pop_back();
+            result = true;
+        }
+    }
+    return result;
+}
+
+void parse_ll_t::accept_token(parse_token_t token)
+{
+    PARSE_ASSERT(token.type >= FIRST_PARSE_TOKEN_TYPE);
+    PARSE_ASSERT(! symbol_stack.empty());
+    bool consumed = false;
+    while (! consumed && ! this->errored)
+    {
+        fprintf(stderr, "Top type %ls\n", token_type_description(stack_top_type()).c_str());
+        if (top_node_match_token(token))
+        {
+            consumed = true;
+            break;
+        }
+        
+        switch (stack_top_type())
+        {
+            /* Symbols */
+            case symbol_statement_list:
+                accept_token_statement_list(token);
+                break;
+                
+            case symbol_statement:
+                accept_token_statement(token);
+                break;
+                
+            case symbol_block_statement:
+                symbol_stack_pop_push(symbol_block_header, symbol_statement_list, parse_keyword_end, symbol_arguments_or_redirections_list);
+                break;
+                
+            case symbol_block_header:
+                accept_token_block_header(token);
+                break;
+                
+            case symbol_if_header:
+                break;
+                
+            case symbol_for_header:
+                symbol_stack_pop_push(parse_keyword_for, parse_token_type_string, parse_keyword_in, symbol_arguments_or_redirections_list, parse_token_type_end);
+                break;
+                
+            case symbol_while_header:
+                symbol_stack_pop_push(parse_keyword_while, symbol_statement);
+                break;
+                
+            case symbol_begin_header:
+                symbol_stack_pop_push(parse_keyword_begin, parse_token_type_end);
+                break;
+                
+            case symbol_function_header:
+                symbol_stack_pop_push(parse_keyword_function, symbol_arguments_or_redirections_list, parse_token_type_end);
+                break;
+                
+            case symbol_boolean_statement:
+                accept_token_boolean_statement(token);
+                break;
+                
+            case symbol_decorated_statement:
+                accept_token_decorated_statement(token);
+                break;
+                
+            case symbol_plain_statement:
+                accept_token_plain_statement(token);
+                break;
+                
+            case symbol_arguments_or_redirections_list:
+                accept_token_arguments_or_redirections_list(token);
+                break;
+                
+            case symbol_argument_or_redirection:
+                accept_token_argument_or_redirection(token);
+                break;
+                
+            /* Tokens */
+            case parse_token_type_string:
+                consumed = accept_token_string(token);
+                break;
+                
+            default:
+                fprintf(stderr, "Bailing with token type %d\n", (int)token.type);
+                break;
+        }
+    }
+}
 
 parse_t::parse_t() : parser(new parse_ll_t())
 {
 }
+
+static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt)
+{
+    parse_keyword_t result = parse_keyword_none;
+    if (tok == TOK_STRING)
+    {
+    
+        const struct {
+            const wchar_t *txt;
+            parse_keyword_t keyword;
+        } keywords[] = {
+            {L"if", parse_keyword_if},
+            {L"else", parse_keyword_else},
+            {L"for", parse_keyword_for},
+            {L"in", parse_keyword_in},
+            {L"while", parse_keyword_while},
+            {L"begin", parse_keyword_begin},
+            {L"function", parse_keyword_function},
+            {L"switch", parse_keyword_switch},
+            {L"end", parse_keyword_end},
+            {L"and", parse_keyword_and},
+            {L"or", parse_keyword_or},
+            {L"not", parse_keyword_not},
+            {L"command", parse_keyword_command},
+            {L"builtin", parse_keyword_builtin}
+        };
+        
+        for (size_t i=0; i < sizeof keywords / sizeof *keywords; i++)
+        {
+            if (! wcscmp(keywords[i].txt, tok_txt))
+            {
+                result = keywords[i].keyword;
+                break;
+            }
+        }
+    }
+    return result;
+}
+
+void parse_t::parse(const wcstring &str)
+{
+    tokenizer_t tok = tokenizer_t(str.c_str(), 0);
+    for (; tok_has_next(&tok); tok_next(&tok))
+    {
+        token_type tok_type = static_cast<token_type>(tok_last_type(&tok));
+        const wchar_t *tok_txt = tok_last(&tok);
+        int tok_start = tok_get_pos(&tok);
+        
+        if (tok_type == TOK_ERROR)
+        {
+            fprintf(stderr, "Tokenizer error\n");
+            break;
+        }
+        
+        parse_token_t token = parse_token_from_tokenizer_token(tok_type);
+        token.tokenizer_type = tok_type;
+        token.source_start = (size_t)tok_start;
+        token.source_length = wcslen(tok_txt);
+        token.keyword = keyword_for_token(tok_type, tok_txt);
+        this->parser->accept_token(token);
+    }
+    wcstring result = dump_tree(this->parser->nodes, str);
+    fprintf(stderr, "Tree (%ld nodes):\n%ls", this->parser->nodes.size(), result.c_str());
+    fprintf(stderr, "node size %ld", sizeof(parse_node_t));
+}

expression.h

@@ -20,9 +20,14 @@ class parse_sr_t;
 class parse_t
 {
     parse_ll_t * const parser;
+    
+    public:
     parse_t();
+    void parse(const wcstring &str);
 };
 
+
+
 /* Fish grammar:
 
 # A statement_list is a list of statements, separated by semicolons or newlines

fish_tests.cpp

@@ -60,6 +60,7 @@
 #include "postfork.h"
 #include "signal.h"
 #include "highlight.h"
+#include "expression.h"
 
 /**
    The number of tests to run
@@ -1717,6 +1718,12 @@ void history_tests_t::test_history_speed(void)
     delete hist;
 }
 
+static void test_new_parser(void)
+{
+    say(L"Testing new parser!");
+    parse_t parser;
+    parser.parse(L"echo hello");
+}
 
 /**
    Main test
@@ -1733,12 +1740,15 @@ int main(int argc, char **argv)
     say(L"Lines beginning with '(ignore):' are not errors, they are warning messages\ngenerated by the fish parser library when given broken input, and can be\nignored. All actual errors begin with 'Error:'.");
     set_main_thread();
     setup_fork_guards();
-    proc_init();
+    //proc_init();
     event_init();
     function_init();
     builtin_init();
     reader_init();
     env_init();
+    
+    test_new_parser();
+    return 0;
 
     test_format();
     test_escape();

tokenizer.h

@@ -19,7 +19,6 @@ enum token_type
 {
     TOK_NONE, /**< Tokenizer not yet constructed */
     TOK_ERROR, /**< Error reading token */
-    TOK_INVALID,/**< Invalid token */
     TOK_STRING,/**< String token */
     TOK_PIPE,/**< Pipe token */
     TOK_END,/**< End token (semicolon or newline, not literal end) */