Merge remote-tracking branch 'upstream/master' into bind_mode

Conflicts:
	builtin.cpp
	reader.cpp
	share/functions/fish_default_key_bindings.fish

Makefile.in

@@ -92,7 +92,7 @@ FISH_OBJS := function.o builtin.o complete.o env.o exec.o expand.o		\
 	env_universal.o env_universal_common.o input_common.o event.o		\
 	signal.o io.o parse_util.o common.o screen.o path.o autoload.o		\
 	parser_keywords.o iothread.o color.o postfork.o	\
-	builtin_test.o
+	builtin_test.o parse_tree.o parse_productions.o parse_execution.cpp
 
 FISH_INDENT_OBJS := fish_indent.o print_help.o common.o	\
 parser_keywords.o wutil.o tokenizer.o
@@ -552,6 +552,8 @@ install-force: all install-translations
 	$(INSTALL) -m 755 -d $(DESTDIR)$(datadir)/fish/man/man1
 	$(INSTALL) -m 755 -d $(DESTDIR)$(datadir)/fish/tools
 	$(INSTALL) -m 755 -d $(DESTDIR)$(datadir)/fish/tools/web_config
+	$(INSTALL) -m 755 -d $(DESTDIR)$(datadir)/fish/tools/web_config/js
+	$(INSTALL) -m 755 -d $(DESTDIR)$(datadir)/fish/tools/web_config/partials
 	$(INSTALL) -m 755 -d $(DESTDIR)$(datadir)/fish/tools/web_config/sample_prompts
 	$(INSTALL) -m 644 etc/config.fish                  $(DESTDIR)$(sysconfdir)/fish/
 	$(INSTALL) -m 644 share/config.fish                $(DESTDIR)$(datadir)/fish/
@@ -575,6 +577,14 @@ install-force: all install-translations
 		$(INSTALL) -m 644 $$i $(DESTDIR)$(datadir)/fish/tools/web_config/; \
 		true; \
 	done;
+	for i in share/tools/web_config/js/*; do\
+		$(INSTALL) -m 644 $$i $(DESTDIR)$(datadir)/fish/tools/web_config/js/; \
+		true; \
+	done;
+	for i in share/tools/web_config/partials/*; do\
+		$(INSTALL) -m 644 $$i $(DESTDIR)$(datadir)/fish/tools/web_config/partials/; \
+		true; \
+	done;
 	for i in share/tools/web_config/sample_prompts/*.fish; do\
 		$(INSTALL) -m 644 $$i $(DESTDIR)$(datadir)/fish/tools/web_config/sample_prompts/; \
 		true; \

autoload.cpp

@@ -195,7 +195,6 @@ autoload_function_t *autoload_t::get_autoloaded_function_with_creation(const wcs
 bool autoload_t::locate_file_and_maybe_load_it(const wcstring &cmd, bool really_load, bool reload, const wcstring_list_t &path_list)
 {
     /* Note that we are NOT locked in this function! */
-    size_t i;
     bool reloaded = 0;
 
     /* Try using a cached function. If we really want the function to be loaded, require that it be really loaded. If we're not reloading, allow stale functions. */
@@ -276,7 +275,7 @@ bool autoload_t::locate_file_and_maybe_load_it(const wcstring &cmd, bool really_
     if (! has_script_source)
     {
         /* Iterate over path searching for suitable completion files */
-        for (i=0; i<path_list.size(); i++)
+        for (size_t i=0; i<path_list.size(); i++)
         {
             wcstring next = path_list.at(i);
             wcstring path = next + L"/" + cmd + L".fish";

builtin.cpp

@@ -64,6 +64,7 @@
 #include "expand.h"
 #include "path.h"
 #include "history.h"
+#include "parse_tree.h"
 
 /**
    The default prompt for the read command
@@ -164,7 +165,7 @@ static const io_chain_t *real_io;
 /**
    Counts the number of non null pointers in the specified array
 */
-static int builtin_count_args(wchar_t **argv)
+static int builtin_count_args(const wchar_t * const * argv)
 {
     int argc = 1;
     while (argv[argc] != NULL)
@@ -243,9 +244,6 @@ wcstring builtin_help_get(parser_t &parser, const wchar_t *name)
 
 static void builtin_print_help(parser_t &parser, const wchar_t *cmd, wcstring &b)
 {
-
-    int is_short = 0;
-
     if (&b == &stderr_buffer)
     {
         stderr_buffer.append(parser.current_line());
@@ -259,7 +257,7 @@ static void builtin_print_help(parser_t &parser, const wchar_t *cmd, wcstring &b
     wchar_t *str = wcsdup(h.c_str());
     if (str)
     {
-
+        bool is_short = false;
         if (&b == &stderr_buffer)
         {
 
@@ -278,7 +276,7 @@ static void builtin_print_help(parser_t &parser, const wchar_t *cmd, wcstring &b
                 int cut=0;
                 int i;
 
-                is_short = 1;
+                is_short = true;
 
                 /*
                   First move down 4 lines
@@ -789,7 +787,6 @@ static int builtin_block(parser_t &parser, wchar_t **argv)
     int scope=UNSET;
     int erase = 0;
     int argc=builtin_count_args(argv);
-    int type = (1<<EVENT_ANY);
 
     woptind=0;
 
@@ -887,7 +884,7 @@ static int builtin_block(parser_t &parser, wchar_t **argv)
         block_t *block = parser.block_at_index(block_idx);
 
         event_blockage_t eb = {};
-        eb.typemask = type;
+        eb.typemask = (1<<EVENT_ANY);
 
         switch (scope)
         {
@@ -1093,20 +1090,22 @@ static int builtin_emit(parser_t &parser, wchar_t **argv)
 static int builtin_generic(parser_t &parser, wchar_t **argv)
 {
     int argc=builtin_count_args(argv);
+
+    /* Hackish - if we have no arguments other than the command, we are a "naked invocation" and we just print help */
+    if (argc == 1)
+    {
+        builtin_print_help(parser, argv[0], stdout_buffer);
+        return STATUS_BUILTIN_ERROR;
+    }
+
     woptind=0;
 
     static const struct woption
             long_options[] =
     {
-        {
-            L"help", no_argument, 0, 'h'
-        }
-        ,
-        {
-            0, 0, 0, 0
-        }
-    }
-    ;
+        { L"help", no_argument, 0, 'h' },
+        { 0, 0, 0, 0 }
+    };
 
     while (1)
     {
@@ -1803,12 +1802,325 @@ static int builtin_pwd(parser_t &parser, wchar_t **argv)
     }
 }
 
+/* This is nearly identical to builtin_function, and is intended to be the successor (with no block manipulation, no function/end split) */
+int define_function(parser_t &parser, const wcstring_list_t &c_args, const wcstring &contents, wcstring *out_err)
+{
+    assert(out_err != NULL);
+
+    /* wgetopt expects 'function' as the first argument. Make a new wcstring_list with that property. */
+    wcstring_list_t args;
+    args.push_back(L"function");
+    args.insert(args.end(), c_args.begin(), c_args.end());
+
+    /* Hackish const_cast matches the one in builtin_run */
+    const null_terminated_array_t<wchar_t> argv_array(args);
+    wchar_t **argv = const_cast<wchar_t **>(argv_array.get());
+
+    int argc = builtin_count_args(argv);
+    int res=STATUS_BUILTIN_OK;
+    wchar_t *desc=0;
+    std::vector<event_t> events;
+    std::auto_ptr<wcstring_list_t> named_arguments(NULL);
+
+    wchar_t *name = 0;
+    bool shadows = true;
+
+    woptind=0;
+
+    const struct woption long_options[] =
+    {
+        { L"description", required_argument, 0, 'd' },
+        { L"on-signal", required_argument, 0, 's' },
+        { L"on-job-exit", required_argument, 0, 'j' },
+        { L"on-process-exit", required_argument, 0, 'p' },
+        { L"on-variable", required_argument, 0, 'v' },
+        { L"on-event", required_argument, 0, 'e' },
+        { L"help", no_argument, 0, 'h' },
+        { L"argument-names", no_argument, 0, 'a' },
+        { L"no-scope-shadowing", no_argument, 0, 'S' },
+        { 0, 0, 0, 0 }
+    };
+
+    while (1 && (!res))
+    {
+        int opt_index = 0;
+
+        int opt = wgetopt_long(argc,
+                               argv,
+                               L"d:s:j:p:v:e:haS",
+                               long_options,
+                               &opt_index);
+        if (opt == -1)
+            break;
+
+        switch (opt)
+        {
+            case 0:
+                if (long_options[opt_index].flag != 0)
+                    break;
+
+
+
+                append_format(*out_err,
+                              BUILTIN_ERR_UNKNOWN,
+                              argv[0],
+                              long_options[opt_index].name);
+
+                res = 1;
+                break;
+
+            case 'd':
+                desc=woptarg;
+                break;
+
+            case 's':
+            {
+                int sig = wcs2sig(woptarg);
+
+                if (sig < 0)
+                {
+                    append_format(*out_err,
+                                  _(L"%ls: Unknown signal '%ls'\n"),
+                                  argv[0],
+                                  woptarg);
+                    res=1;
+                    break;
+                }
+                events.push_back(event_t::signal_event(sig));
+                break;
+            }
+
+            case 'v':
+            {
+                if (wcsvarname(woptarg))
+                {
+                    append_format(*out_err,
+                                  _(L"%ls: Invalid variable name '%ls'\n"),
+                                  argv[0],
+                                  woptarg);
+                    res=STATUS_BUILTIN_ERROR;
+                    break;
+                }
+
+                events.push_back(event_t::variable_event(woptarg));
+                break;
+            }
+
+
+            case 'e':
+            {
+                events.push_back(event_t::generic_event(woptarg));
+                break;
+            }
+
+            case 'j':
+            case 'p':
+            {
+                pid_t pid;
+                wchar_t *end;
+                event_t e(EVENT_ANY);
+
+                if ((opt == 'j') &&
+                        (wcscasecmp(woptarg, L"caller") == 0))
+                {
+                    int job_id = -1;
+
+                    if (is_subshell)
+                    {
+                        size_t block_idx = 0;
+
+                        /* Find the outermost substitution block */
+                        for (block_idx = 0; ; block_idx++)
+                        {
+                            const block_t *b = parser.block_at_index(block_idx);
+                            if (b == NULL || b->type() == SUBST)
+                                break;
+                        }
+
+                        /* Go one step beyond that, to get to the caller */
+                        const block_t *caller_block = parser.block_at_index(block_idx + 1);
+                        if (caller_block != NULL && caller_block->job != NULL)
+                        {
+                            job_id = caller_block->job->job_id;
+                        }
+                    }
+
+                    if (job_id == -1)
+                    {
+                        append_format(*out_err,
+                                      _(L"%ls: Cannot find calling job for event handler\n"),
+                                      argv[0]);
+                        res=1;
+                    }
+                    else
+                    {
+                        e.type = EVENT_JOB_ID;
+                        e.param1.job_id = job_id;
+                    }
+
+                }
+                else
+                {
+                    errno = 0;
+                    pid = fish_wcstoi(woptarg, &end, 10);
+                    if (errno || !end || *end)
+                    {
+                        append_format(*out_err,
+                                      _(L"%ls: Invalid process id %ls\n"),
+                                      argv[0],
+                                      woptarg);
+                        res=1;
+                        break;
+                    }
+
+
+                    e.type = EVENT_EXIT;
+                    e.param1.pid = (opt=='j'?-1:1)*abs(pid);
+                }
+                if (res)
+                {
+                    /* nothing */
+                }
+                else
+                {
+                    events.push_back(e);
+                }
+                break;
+            }
+
+            case 'a':
+                if (named_arguments.get() == NULL)
+                    named_arguments.reset(new wcstring_list_t);
+                break;
+
+            case 'S':
+                shadows = 0;
+                break;
+
+            case 'h':
+                builtin_print_help(parser, argv[0], stdout_buffer);
+                return STATUS_BUILTIN_OK;
+
+            case '?':
+                builtin_unknown_option(parser, argv[0], argv[woptind-1]);
+                res = 1;
+                break;
+
+        }
+
+    }
+
+    if (!res)
+    {
+
+        if (argc == woptind)
+        {
+            append_format(*out_err,
+                          _(L"%ls: Expected function name\n"),
+                          argv[0]);
+            res=1;
+        }
+        else if (wcsfuncname(argv[woptind]))
+        {
+            append_format(*out_err,
+                          _(L"%ls: Illegal function name '%ls'\n"),
+                          argv[0],
+                          argv[woptind]);
+
+            res=1;
+        }
+        else if (parser_keywords_is_reserved(argv[woptind]))
+        {
+
+            append_format(*out_err,
+                          _(L"%ls: The name '%ls' is reserved,\nand can not be used as a function name\n"),
+                          argv[0],
+                          argv[woptind]);
+
+            res=1;
+        }
+        else if (! wcslen(argv[woptind]))
+        {
+            append_format(*out_err, _(L"%ls: No function name given\n"), argv[0]);
+        }
+        else
+        {
+
+            name = argv[woptind++];
+
+            if (named_arguments.get())
+            {
+                while (woptind < argc)
+                {
+                    if (wcsvarname(argv[woptind]))
+                    {
+                        append_format(*out_err,
+                                      _(L"%ls: Invalid variable name '%ls'\n"),
+                                      argv[0],
+                                      argv[woptind]);
+                        res = STATUS_BUILTIN_ERROR;
+                        break;
+                    }
+
+                    named_arguments->push_back(argv[woptind++]);
+                }
+            }
+            else if (woptind != argc)
+            {
+                append_format(*out_err,
+                              _(L"%ls: Expected one argument, got %d\n"),
+                              argv[0],
+                              argc);
+                res=1;
+
+            }
+        }
+    }
+
+    if (res)
+    {
+        builtin_print_help(parser, argv[0], *out_err);
+    }
+    else
+    {
+        function_data_t d;
+
+        d.name = name;
+        if (desc)
+            d.description = desc;
+        d.events.swap(events);
+        d.shadows = shadows;
+        if (named_arguments.get())
+            d.named_arguments.swap(*named_arguments);
+
+        for (size_t i=0; i<d.events.size(); i++)
+        {
+            event_t &e = d.events.at(i);
+            e.function_name = d.name;
+        }
+
+        d.definition = contents.c_str();
+
+        // TODO: fix def_offset inside function_add
+        function_add(d, parser);
+    }
+
+    return res;
+}
+
 /**
    The function builtin, used for providing subroutines.
    It calls various functions from function.c to perform any heavy lifting.
 */
 static int builtin_function(parser_t &parser, wchar_t **argv)
 {
+    /* Hack hack hack - with the new parser, this is only invoked for help */
+    if (parser_use_ast())
+    {
+        builtin_print_help(parser, argv[0], stdout_buffer);
+        return STATUS_BUILTIN_OK;
+    }
+
     int argc = builtin_count_args(argv);
     int res=STATUS_BUILTIN_OK;
     wchar_t *desc=0;
@@ -1921,7 +2233,7 @@ static int builtin_function(parser_t &parser, wchar_t **argv)
                     if (is_subshell)
                     {
                         size_t block_idx = 0;
-                        
+
                         /* Find the outermost substitution block */
                         for (block_idx = 0; ; block_idx++)
                         {
@@ -1929,7 +2241,7 @@ static int builtin_function(parser_t &parser, wchar_t **argv)
                             if (b == NULL || b->type() == SUBST)
                                 break;
                         }
-                        
+
                         /* Go one step beyond that, to get to the caller */
                         const block_t *caller_block = parser.block_at_index(block_idx + 1);
                         if (caller_block != NULL && caller_block->job != NULL)
@@ -2034,6 +2346,10 @@ static int builtin_function(parser_t &parser, wchar_t **argv)
 
             res=1;
         }
+        else if (! wcslen(argv[woptind]))
+        {
+            append_format(stderr_buffer, _(L"%ls: No function name given\n"), argv[0]);
+        }
         else
         {
 
@@ -3070,10 +3386,7 @@ static int builtin_source(parser_t &parser, wchar_t ** argv)
 
     argc = builtin_count_args(argv);
 
-    const wchar_t *fn;
-    const wchar_t *fn_intern;
-
-
+    const wchar_t *fn, *fn_intern;
 
     if (argc < 2 || (wcscmp(argv[1], L"-") == 0))
     {
@@ -3106,18 +3419,7 @@ static int builtin_source(parser_t &parser, wchar_t ** argv)
             return STATUS_BUILTIN_ERROR;
         }
 
-        fn = wrealpath(argv[1], NULL);
-
-        if (!fn)
-        {
-            fn_intern = intern(argv[1]);
-        }
-        else
-        {
-            fn_intern = intern(fn);
-            free((void *)fn);
-        }
-
+        fn_intern = intern(argv[1]);
     }
 
     parser.push_block(new source_block_t(fn_intern));
@@ -3421,6 +3723,12 @@ static int builtin_for(parser_t &parser, wchar_t **argv)
     int argc = builtin_count_args(argv);
     int res=STATUS_BUILTIN_ERROR;
 
+    /* Hackish - if we have no arguments other than the command, we are a "naked invocation" and we just print help */
+    if (argc == 1)
+    {
+        builtin_print_help(parser, argv[0], stdout_buffer);
+        return STATUS_BUILTIN_ERROR;
+    }
 
     if (argc < 3)
     {
@@ -3709,7 +4017,7 @@ static int builtin_break_continue(parser_t &parser, wchar_t **argv)
     {
         parser.block_at_index(block_idx)->skip = true;
     }
-    
+
     /* Skip the loop itself */
     block_t *loop_block = parser.block_at_index(loop_idx);
     loop_block->skip = true;
@@ -3787,7 +4095,7 @@ static int builtin_return(parser_t &parser, wchar_t **argv)
         builtin_print_help(parser, argv[0], stderr_buffer);
         return STATUS_BUILTIN_ERROR;
     }
-    
+
     /* Skip everything up to (and then including) the function block */
     for (size_t i=0; i < function_block_idx; i++)
     {
@@ -3808,6 +4116,13 @@ static int builtin_switch(parser_t &parser, wchar_t **argv)
     int res=STATUS_BUILTIN_OK;
     int argc = builtin_count_args(argv);
 
+    /* Hackish - if we have no arguments other than the command, we are a "naked invocation" and we just print help */
+    if (argc == 1)
+    {
+        builtin_print_help(parser, argv[0], stdout_buffer);
+        return STATUS_BUILTIN_ERROR;
+    }
+
     if (argc != 2)
     {
         append_format(stderr_buffer,
@@ -4012,6 +4327,49 @@ static int builtin_history(parser_t &parser, wchar_t **argv)
     return STATUS_BUILTIN_ERROR;
 }
 
+#pragma mark Simulator
+
+int builtin_parse(parser_t &parser, wchar_t **argv)
+{
+    struct sigaction act;
+    sigemptyset(& act.sa_mask);
+    act.sa_flags=0;
+    act.sa_handler=SIG_DFL;
+    sigaction(SIGINT, &act, 0);
+
+    std::vector<char> txt;
+    for (;;)
+    {
+        char buff[256];
+        ssize_t amt = read_loop(builtin_stdin, buff, sizeof buff);
+        if (amt <= 0) break;
+        txt.insert(txt.end(), buff, buff + amt);
+    }
+    if (! txt.empty())
+    {
+        const wcstring src = str2wcstring(&txt.at(0), txt.size());
+        parse_node_tree_t parse_tree;
+        parse_error_list_t errors;
+        bool success = parse_tree_from_string(src, parse_flag_none, &parse_tree, &errors);
+        if (! success)
+        {
+            stdout_buffer.append(L"Parsing failed:\n");
+            for (size_t i=0; i < errors.size(); i++)
+            {
+                stdout_buffer.append(errors.at(i).describe(src));
+                stdout_buffer.push_back(L'\n');
+            }
+
+            stdout_buffer.append(L"(Reparsed with continue after error)\n");
+            parse_tree.clear();
+            errors.clear();
+            parse_tree_from_string(src, parse_flag_continue_after_error, &parse_tree, &errors);
+        }
+        const wcstring dump = parse_dump_tree(parse_tree, src);
+        stdout_buffer.append(dump);
+    }
+    return STATUS_BUILTIN_OK;
+}
 
 /*
   END OF BUILTIN COMMANDS
@@ -4027,6 +4385,7 @@ static int builtin_history(parser_t &parser, wchar_t **argv)
 static const builtin_data_t builtin_datas[]=
 {
     { 		L"[",  &builtin_test, N_(L"Test a condition")   },
+    { 		L"__fish_parse",  &builtin_parse, N_(L"Try out the new parser")  },
     { 		L"and",  &builtin_generic, N_(L"Execute command if previous command suceeded")  },
     { 		L"begin",  &builtin_begin, N_(L"Create a block of code")   },
     { 		L"bg",  &builtin_bg, N_(L"Send job to background")   },
@@ -4171,7 +4530,7 @@ void builtin_get_names(std::vector<completion_t> &list)
 {
     for (size_t i=0; i < BUILTIN_COUNT; i++)
     {
-        list.push_back(completion_t(builtin_datas[i].name));
+        append_completion(list, builtin_datas[i].name);
     }
 }
 
@@ -4218,4 +4577,3 @@ void builtin_pop_io(parser_t &parser)
         builtin_stdin = 0;
     }
 }
-

builtin.h

@@ -176,7 +176,10 @@ const wchar_t *builtin_complete_get_temporary_buffer();
    Run the __fish_print_help function to obtain the help information
    for the specified command.
 */
-
 wcstring builtin_help_get(parser_t &parser, const wchar_t *cmd);
 
+/** Defines a function, like builtin_function. Returns 0 on success. args should NOT contain 'function' as the first argument. */
+int define_function(parser_t &parser, const wcstring_list_t &args, const wcstring &contents, wcstring *out_err);
+
+
 #endif

builtin_commandline.cpp

@@ -143,17 +143,13 @@ static void write_part(const wchar_t *begin,
                        int cut_at_cursor,
                        int tokenize)
 {
-    wcstring out;
-    wchar_t *buff;
-    size_t pos;
-
-    pos = get_cursor_pos()-(begin-get_buffer());
+    size_t pos = get_cursor_pos()-(begin-get_buffer());
 
     if (tokenize)
     {
-        buff = wcsndup(begin, end-begin);
+        wchar_t *buff = wcsndup(begin, end-begin);
 //    fwprintf( stderr, L"Subshell: %ls, end char %lc\n", buff, *end );
-        out.clear();
+        wcstring out;
         tokenizer_t tok(buff, TOK_ACCEPT_UNFINISHED);
         for (; tok_has_next(&tok); tok_next(&tok))
         {

builtin_complete.cpp

@@ -290,7 +290,6 @@ static int builtin_complete(parser_t &parser, wchar_t **argv)
     int result_mode=SHARED;
     int remove = 0;
     int authoritative = -1;
-    int flags = COMPLETE_AUTO_SPACE;
 
     wcstring short_opt;
     wcstring_list_t gnu_opt, old_opt;
@@ -497,15 +496,19 @@ static int builtin_complete(parser_t &parser, wchar_t **argv)
     {
         if (condition && wcslen(condition))
         {
-            if (parser.test(condition))
+            const wcstring condition_string = condition;
+            parse_error_list_t errors;
+            if (parse_util_detect_errors(condition_string, &errors))
             {
                 append_format(stderr_buffer,
-                              L"%ls: Condition '%ls' contained a syntax error\n",
+                              L"%ls: Condition '%ls' contained a syntax error",
                               argv[0],
                               condition);
-
-                parser.test(condition, NULL, &stderr_buffer, argv[0]);
-
+                for (size_t i=0; i < errors.size(); i++)
+                {
+                    append_format(stderr_buffer, L"\n%s: ", argv[0]);
+                    stderr_buffer.append(errors.at(i).describe(condition_string));
+                }
                 res = true;
             }
         }
@@ -596,6 +599,8 @@ static int builtin_complete(parser_t &parser, wchar_t **argv)
         }
         else
         {
+            int flags = COMPLETE_AUTO_SPACE;
+
             if (remove)
             {
                 builtin_complete_remove(cmd,

builtin_jobs.cpp

@@ -164,7 +164,6 @@ static int builtin_jobs(parser_t &parser, wchar_t **argv)
     int found=0;
     int mode=JOBS_DEFAULT;
     int print_last = 0;
-    const job_t *j;
 
     argc = builtin_count_args(argv);
     woptind=0;
@@ -305,7 +304,7 @@ static int builtin_jobs(parser_t &parser, wchar_t **argv)
                     return 1;
                 }
 
-                j = job_get_from_pid(pid);
+                const job_t *j = job_get_from_pid(pid);
 
                 if (j && !job_is_completed(j))
                 {

builtin_set.cpp

@@ -697,7 +697,6 @@ static int builtin_set(parser_t &parser, wchar_t **argv)
         /*
           Slice mode
         */
-        size_t idx_count, val_count;
         std::vector<long> indexes;
         wcstring_list_t result;
 
@@ -714,8 +713,8 @@ static int builtin_set(parser_t &parser, wchar_t **argv)
                 break;
             }
 
-            val_count = argc-woptind-1;
-            idx_count = indexes.size();
+            size_t idx_count = indexes.size();
+            size_t val_count = argc-woptind-1;
 
             if (!erase)
             {

common.cpp

@@ -105,7 +105,7 @@ void show_stackframe()
         return;
 
     void *trace[32];
-    int i, trace_size = 0;
+    int trace_size = 0;
 
     trace_size = backtrace(trace, 32);
     char **messages = backtrace_symbols(trace, trace_size);
@@ -113,7 +113,7 @@ void show_stackframe()
     if (messages)
     {
         debug(0, L"Backtrace:");
-        for (i=0; i<trace_size; i++)
+        for (int i=0; i<trace_size; i++)
         {
             fwprintf(stderr, L"%s\n", messages[i]);
         }
@@ -508,7 +508,7 @@ const wchar_t *wcsfuncname(const wchar_t *str)
 }
 
 
-int wcsvarchr(wchar_t chr)
+bool wcsvarchr(wchar_t chr)
 {
     return iswalnum(chr) || chr == L'_';
 }
@@ -761,7 +761,7 @@ void debug_safe(int level, const char *msg, const char *param1, const char *para
     errno = errno_old;
 }
 
-void format_long_safe(char buff[128], long val)
+void format_long_safe(char buff[64], long val)
 {
     if (val == 0)
     {
@@ -795,7 +795,7 @@ void format_long_safe(char buff[128], long val)
     }
 }
 
-void format_long_safe(wchar_t buff[128], long val)
+void format_long_safe(wchar_t buff[64], long val)
 {
     if (val == 0)
     {
@@ -830,19 +830,18 @@ void format_long_safe(wchar_t buff[128], long val)
 
 void write_screen(const wcstring &msg, wcstring &buff)
 {
-    const wchar_t *start, *pos;
     int line_width = 0;
-    int tok_width = 0;
     int screen_width = common_get_width();
 
     if (screen_width)
     {
-        start = pos = msg.c_str();
+        const wchar_t *start = msg.c_str();
+        const wchar_t *pos = start;
         while (1)
         {
             int overflow = 0;
 
-            tok_width=0;
+            int tok_width=0;
 
             /*
               Tokenize on whitespace, and also calculate the width of the token
@@ -917,61 +916,28 @@ void write_screen(const wcstring &msg, wcstring &buff)
     buff.push_back(L'\n');
 }
 
-/**
-   Perform string escaping of a strinng by only quoting it. Assumes
-   the string has already been checked for characters that can not be
-   escaped this way.
- */
-static wchar_t *escape_simple(const wchar_t *in)
+/* Escape a string, storing the result in out_str */
+static void escape_string_internal(const wchar_t *orig_in, size_t in_len, wcstring *out_str, escape_flags_t flags)
 {
-    wchar_t *out;
-    size_t len = wcslen(in);
-    out = (wchar_t *)malloc(sizeof(wchar_t)*(len+3));
-    if (!out)
-        DIE_MEM();
-
-    out[0] = L'\'';
-    wcscpy(&out[1], in);
-    out[len+1]=L'\'';
-    out[len+2]=0;
-    return out;
-}
-
-wchar_t *escape(const wchar_t *in_orig, escape_flags_t flags)
-{
-    const wchar_t *in = in_orig;
+    assert(orig_in != NULL);
 
+    const wchar_t *in = orig_in;
     bool escape_all = !!(flags & ESCAPE_ALL);
     bool no_quoted  = !!(flags & ESCAPE_NO_QUOTED);
     bool no_tilde = !!(flags & ESCAPE_NO_TILDE);
 
-    wchar_t *out;
-    wchar_t *pos;
-
     int need_escape=0;
     int need_complex_escape=0;
 
-    if (!in)
+    /* Avoid dereferencing all over the place */
+    wcstring &out = *out_str;
+
+    if (!no_quoted && in_len == 0)
     {
-        debug(0, L"%s called with null input", __func__);
-        FATAL_EXIT();
+        out.assign(L"''");
+        return;
     }
 
-    if (!no_quoted && (wcslen(in) == 0))
-    {
-        out = wcsdup(L"''");
-        if (!out)
-            DIE_MEM();
-        return out;
-    }
-
-
-    out = (wchar_t *)malloc(sizeof(wchar_t)*(wcslen(in)*4 + 1));
-    pos = out;
-
-    if (!out)
-        DIE_MEM();
-
     while (*in != 0)
     {
 
@@ -981,14 +947,14 @@ wchar_t *escape(const wchar_t *in_orig, escape_flags_t flags)
             int val = *in - ENCODE_DIRECT_BASE;
             int tmp;
 
-            *(pos++) = L'\\';
-            *(pos++) = L'X';
+            out += L'\\';
+            out += L'X';
 
             tmp = val/16;
-            *pos++ = tmp > 9? L'a'+(tmp-10):L'0'+tmp;
+            out += tmp > 9? L'a'+(tmp-10):L'0'+tmp;
 
             tmp = val%16;
-            *pos++ = tmp > 9? L'a'+(tmp-10):L'0'+tmp;
+            out += tmp > 9? L'a'+(tmp-10):L'0'+tmp;
             need_escape=need_complex_escape=1;
 
         }
@@ -998,32 +964,32 @@ wchar_t *escape(const wchar_t *in_orig, escape_flags_t flags)
             switch (c)
             {
                 case L'\t':
-                    *(pos++) = L'\\';
-                    *(pos++) = L't';
+                    out += L'\\';
+                    out += L't';
                     need_escape=need_complex_escape=1;
                     break;
 
                 case L'\n':
-                    *(pos++) = L'\\';
-                    *(pos++) = L'n';
+                    out += L'\\';
+                    out += L'n';
                     need_escape=need_complex_escape=1;
                     break;
 
                 case L'\b':
-                    *(pos++) = L'\\';
-                    *(pos++) = L'b';
+                    out += L'\\';
+                    out += L'b';
                     need_escape=need_complex_escape=1;
                     break;
 
                 case L'\r':
-                    *(pos++) = L'\\';
-                    *(pos++) = L'r';
+                    out += L'\\';
+                    out += L'r';
                     need_escape=need_complex_escape=1;
                     break;
 
                 case L'\x1b':
-                    *(pos++) = L'\\';
-                    *(pos++) = L'e';
+                    out += L'\\';
+                    out += L'e';
                     need_escape=need_complex_escape=1;
                     break;
 
@@ -1033,8 +999,8 @@ wchar_t *escape(const wchar_t *in_orig, escape_flags_t flags)
                 {
                     need_escape=need_complex_escape=1;
                     if (escape_all)
-                        *pos++ = L'\\';
-                    *pos++ = *in;
+                        out += L'\\';
+                    out += *in;
                     break;
                 }
 
@@ -1063,9 +1029,9 @@ wchar_t *escape(const wchar_t *in_orig, escape_flags_t flags)
                     {
                         need_escape=1;
                         if (escape_all)
-                            *pos++ = L'\\';
+                            out += L'\\';
                     }
-                    *pos++ = *in;
+                    out += *in;
                     break;
                 }
 
@@ -1075,9 +1041,9 @@ wchar_t *escape(const wchar_t *in_orig, escape_flags_t flags)
                     {
                         if (*in <27 && *in > 0)
                         {
-                            *(pos++) = L'\\';
-                            *(pos++) = L'c';
-                            *(pos++) = L'a' + *in -1;
+                            out += L'\\';
+                            out += L'c';
+                            out += L'a' + *in -1;
 
                             need_escape=need_complex_escape=1;
                             break;
@@ -1086,15 +1052,15 @@ wchar_t *escape(const wchar_t *in_orig, escape_flags_t flags)
 
 
                         int tmp = (*in)%16;
-                        *pos++ = L'\\';
-                        *pos++ = L'x';
-                        *pos++ = ((*in>15)? L'1' : L'0');
-                        *pos++ = tmp > 9? L'a'+(tmp-10):L'0'+tmp;
+                        out += L'\\';
+                        out += L'x';
+                        out += ((*in>15)? L'1' : L'0');
+                        out += tmp > 9? L'a'+(tmp-10):L'0'+tmp;
                         need_escape=need_complex_escape=1;
                     }
                     else
                     {
-                        *pos++ = *in;
+                        out += *in;
                     }
                     break;
                 }
@@ -1103,7 +1069,6 @@ wchar_t *escape(const wchar_t *in_orig, escape_flags_t flags)
 
         in++;
     }
-    *pos = 0;
 
     /*
       Use quoted escaping if possible, since most people find it
@@ -1111,18 +1076,32 @@ wchar_t *escape(const wchar_t *in_orig, escape_flags_t flags)
      */
     if (!no_quoted && need_escape && !need_complex_escape && escape_all)
     {
-        free(out);
-        out = escape_simple(in_orig);
+        wchar_t single_quote = L'\'';
+        out.clear();
+        out.reserve(2 + in_len);
+        out.push_back(single_quote);
+        out.append(orig_in, in_len);
+        out.push_back(single_quote);
+    }
+}
+
+wchar_t *escape(const wchar_t *in, escape_flags_t flags)
+{
+    if (!in)
+    {
+        debug(0, L"%s called with null input", __func__);
+        FATAL_EXIT();
     }
 
-    return out;
+    wcstring tmp;
+    escape_string_internal(in, wcslen(in), &tmp, flags);
+    return wcsdup(tmp.c_str());
 }
 
 wcstring escape_string(const wcstring &in, escape_flags_t flags)
 {
-    wchar_t *tmp = escape(in.c_str(), flags);
-    wcstring result(tmp);
-    free(tmp);
+    wcstring result;
+    escape_string_internal(in.c_str(), in.size(), &result, flags);
     return result;
 }
 
@@ -1151,7 +1130,7 @@ static size_t read_unquoted_escape(const wchar_t *input, wcstring *result, bool
     switch (c)
     {
 
-        /* A null character after a backslash is an error */
+            /* A null character after a backslash is an error */
         case L'\0':
         {
             /* Adjust in_pos to only include the backslash */
@@ -1198,11 +1177,11 @@ static size_t read_unquoted_escape(const wchar_t *input, wcstring *result, bool
                 {
                     chars=8;
                     max_val = WCHAR_MAX;
-                    
+
                     // Don't exceed the largest Unicode code point - see #1107
                     if (0x10FFFF < max_val)
                         max_val = (wchar_t)0x10FFFF;
-                    
+
                     break;
                 }
 

common.h

@@ -348,8 +348,8 @@ void format_size_safe(char buff[128], unsigned long long sz);
 void debug_safe(int level, const char *msg, const char *param1 = NULL, const char *param2 = NULL, const char *param3 = NULL, const char *param4 = NULL, const char *param5 = NULL, const char *param6 = NULL, const char *param7 = NULL, const char *param8 = NULL, const char *param9 = NULL, const char *param10 = NULL, const char *param11 = NULL, const char *param12 = NULL);
 
 /** Writes out a long safely */
-void format_long_safe(char buff[128], long val);
-void format_long_safe(wchar_t buff[128], long val);
+void format_long_safe(char buff[64], long val);
+void format_long_safe(wchar_t buff[64], long val);
 
 
 template<typename T>
@@ -612,10 +612,10 @@ const wchar_t *wcsfuncname(const wchar_t *str);
 /**
    Test if the given string is valid in a variable name
 
-   \return 1 if this is a valid name, 0 otherwise
+   \return true if this is a valid name, false otherwise
 */
 
-int wcsvarchr(wchar_t chr);
+bool wcsvarchr(wchar_t chr);
 
 
 /**

complete.cpp

@@ -44,6 +44,7 @@
 #include "parser_keywords.h"
 #include "wutil.h"
 #include "path.h"
+#include "parse_tree.h"
 #include "iothread.h"
 
 /*
@@ -280,21 +281,26 @@ completion_t::~completion_t()
 {
 }
 
-/* completion_t functions */
-completion_t::completion_t(const wcstring &comp, const wcstring &desc, string_fuzzy_match_t mat, int flags_val) :
-    completion(comp),
-    description(desc),
-    match(mat),
-    flags(flags_val)
+/* Clear the COMPLETE_AUTO_SPACE flag, and set COMPLETE_NO_SPACE appropriately depending on the suffix of the string */
+static complete_flags_t resolve_auto_space(const wcstring &comp, complete_flags_t flags)
 {
     if (flags & COMPLETE_AUTO_SPACE)
     {
         flags = flags & ~COMPLETE_AUTO_SPACE;
-        size_t len = completion.size();
+        size_t len = comp.size();
         if (len > 0  && (wcschr(L"/=@:", comp.at(len-1)) != 0))
             flags |= COMPLETE_NO_SPACE;
     }
+    return flags;
+}
 
+/* completion_t functions. Note that the constructor resolves flags! */
+completion_t::completion_t(const wcstring &comp, const wcstring &desc, string_fuzzy_match_t mat, complete_flags_t flags_val) :
+    completion(comp),
+    description(desc),
+    match(mat),
+    flags(resolve_auto_space(comp, flags_val))
+{
 }
 
 completion_t::completion_t(const completion_t &him) : completion(him.completion), description(him.description), match(him.match), flags(him.flags)
@@ -462,10 +468,18 @@ void completion_autoload_t::command_removed(const wcstring &cmd)
 }
 
 
-/** Create a new completion entry */
+/** Create a new completion entry. */
 void append_completion(std::vector<completion_t> &completions, const wcstring &comp, const wcstring &desc, complete_flags_t flags, string_fuzzy_match_t match)
 {
-    completions.push_back(completion_t(comp, desc, match, flags));
+    /* If we just constructed the completion and used push_back, we would get two string copies. Try to avoid that by making a stubby completion in the vector first, and then copying our string in. Note that completion_t's constructor will munge 'flags' so it's important that we pass those to the constructor.
+
+       Nasty hack for #1241 - since the constructor needs the completion string to resolve AUTO_SPACE, and we aren't providing it with the completion, we have to do the resolution ourselves. We should get this resolving out of the constructor.
+    */
+    const wcstring empty;
+    completions.push_back(completion_t(empty, empty, match, resolve_auto_space(comp, flags)));
+    completion_t *last = &completions.back();
+    last->completion = comp;
+    last->description = desc;
 }
 
 /**
@@ -1190,7 +1204,7 @@ void completer_t::complete_cmd(const wcstring &str_cmd, bool use_function, bool
             wcstring_list_t names = function_get_names(str_cmd.at(0) == L'_');
             for (size_t i=0; i < names.size(); i++)
             {
-                possible_comp.push_back(completion_t(names.at(i)));
+                append_completion(possible_comp, names.at(i));
             }
 
             this->complete_strings(str_cmd, 0, &complete_function_desc, possible_comp, 0);
@@ -1229,7 +1243,7 @@ void completer_t::complete_from_args(const wcstring &str,
     std::vector<completion_t> possible_comp;
 
     bool is_autosuggest = (this->type() == COMPLETE_AUTOSUGGEST);
-    parser_t parser(is_autosuggest ? PARSER_TYPE_COMPLETIONS_ONLY : PARSER_TYPE_GENERAL, false);
+    parser_t parser(is_autosuggest ? PARSER_TYPE_COMPLETIONS_ONLY : PARSER_TYPE_GENERAL, false /* don't show errors */);
 
     /* If type is COMPLETE_AUTOSUGGEST, it means we're on a background thread, so don't call proc_push_interactive */
     if (! is_autosuggest)
@@ -1360,7 +1374,9 @@ struct local_options_t
 bool completer_t::complete_param(const wcstring &scmd_orig, const wcstring &spopt, const wcstring &sstr, bool use_switches)
 {
 
-    const wchar_t * const cmd_orig = scmd_orig.c_str(), * const popt = spopt.c_str(), * const str = sstr.c_str();
+    const wchar_t * const cmd_orig = scmd_orig.c_str();
+    const wchar_t * const popt = spopt.c_str();
+    const wchar_t * const str = sstr.c_str();
 
     bool use_common=1, use_files=1;
 
@@ -1475,7 +1491,7 @@ bool completer_t::complete_param(const wcstring &scmd_orig, const wcstring &spop
                         {
                             if (o->result_mode & NO_COMMON) use_common = false;
                             if (o->result_mode & NO_FILES) use_files = false;
-                            complete_from_args(str, o->comp.c_str(), o->localized_desc(), o->flags);
+                            complete_from_args(str, o->comp, o->localized_desc(), o->flags);
 
                         }
                     }
@@ -1688,7 +1704,7 @@ bool completer_t::complete_variable(const wcstring &str, size_t start_offset)
                 desc = format_string(COMPLETE_VAR_DESC_VAL, value.c_str());
         }
 
-        append_completion(this->completions,  comp.c_str(), desc.c_str(), flags, match);
+        append_completion(this->completions,  comp, desc, flags, match);
 
         res = true;
     }
@@ -1788,223 +1804,152 @@ bool completer_t::try_complete_user(const wcstring &str)
     return res;
 }
 
-void complete(const wcstring &cmd, std::vector<completion_t> &comps, completion_request_flags_t flags)
+void complete(const wcstring &cmd_with_subcmds, std::vector<completion_t> &comps, completion_request_flags_t flags)
 {
+    /* Determine the innermost subcommand */
+    const wchar_t *cmdsubst_begin, *cmdsubst_end;
+    parse_util_cmdsubst_extent(cmd_with_subcmds.c_str(), cmd_with_subcmds.size(), &cmdsubst_begin, &cmdsubst_end);
+    assert(cmdsubst_begin != NULL && cmdsubst_end != NULL && cmdsubst_end >= cmdsubst_begin);
+    const wcstring cmd = wcstring(cmdsubst_begin, cmdsubst_end - cmdsubst_begin);
+
     /* Make our completer */
     completer_t completer(cmd, flags);
 
-    const wchar_t *tok_begin, *tok_end, *cmdsubst_begin, *cmdsubst_end, *prev_begin, *prev_end;
-    wcstring current_token, prev_token;
     wcstring current_command;
-    int on_command=0;
-    size_t pos;
+    const size_t pos = cmd.size();
     bool done=false;
-    int use_command = 1;
-    int use_function = 1;
-    int use_builtin = 1;
-    int had_ddash = 0;
+    bool use_command = 1;
+    bool use_function = 1;
+    bool use_builtin = 1;
 
-//  debug( 1, L"Complete '%ls'", cmd );
-
-    size_t cursor_pos = cmd.size();
+    //  debug( 1, L"Complete '%ls'", cmd );
 
     const wchar_t *cmd_cstr = cmd.c_str();
-    parse_util_cmdsubst_extent(cmd_cstr, cursor_pos, &cmdsubst_begin, &cmdsubst_end);
-    parse_util_token_extent(cmd_cstr, cursor_pos, &tok_begin, &tok_end, &prev_begin, &prev_end);
-
-    if (!cmdsubst_begin)
-        done=1;
-
+    const wchar_t *tok_begin = NULL, *prev_begin = NULL, *prev_end = NULL;
+    parse_util_token_extent(cmd_cstr, cmd.size(), &tok_begin, NULL, &prev_begin, &prev_end);
 
     /**
-       If we are completing a variable name or a tilde expansion user
-       name, we do that and return. No need for any other completions.
-    */
+     If we are completing a variable name or a tilde expansion user
+     name, we do that and return. No need for any other completions.
+     */
+
+    const wcstring current_token = tok_begin;
 
     if (!done)
     {
-        wcstring tmp = tok_begin;
-        done = completer.try_complete_variable(tmp) || completer.try_complete_user(tmp);
+        done = completer.try_complete_variable(current_token) || completer.try_complete_user(current_token);
     }
 
     if (!done)
     {
-        pos = cursor_pos-(cmdsubst_begin-cmd_cstr);
+        //const size_t prev_token_len = (prev_begin ? prev_end - prev_begin : 0);
+        //const wcstring prev_token(prev_begin, prev_token_len);
 
-        const wcstring buff = wcstring(cmdsubst_begin, cmdsubst_end-cmdsubst_begin);
+        parse_node_tree_t tree;
+        parse_tree_from_string(cmd, parse_flag_continue_after_error | parse_flag_accept_incomplete_tokens, &tree, NULL);
 
-        int had_cmd=0;
-        int end_loop=0;
-
-        tokenizer_t tok(buff.c_str(), TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
-        while (tok_has_next(&tok) && !end_loop)
+        /* Find the plain statement that contains the position */
+        const parse_node_t *plain_statement = tree.find_node_matching_source_location(symbol_plain_statement, pos, NULL);
+        if (plain_statement != NULL)
         {
-            switch (tok_last_type(&tok))
+            assert(plain_statement->has_source() && plain_statement->type == symbol_plain_statement);
+
+            /* Get the command node */
+            const parse_node_t *cmd_node = tree.get_child(*plain_statement, 0, parse_token_type_string);
+
+            /* Get the actual command string */
+            if (cmd_node != NULL)
+                current_command = cmd_node->get_source(cmd);
+
+            /* Check the decoration */
+            switch (tree.decoration_for_plain_statement(*plain_statement))
             {
+                case parse_statement_decoration_none:
+                    use_command = true;
+                    use_function = true;
+                    use_builtin = true;
+                    break;
 
-                case TOK_STRING:
+                case parse_statement_decoration_command:
+                    use_command = true;
+                    use_function = false;
+                    use_builtin = false;
+                    break;
+
+                case parse_statement_decoration_builtin:
+                    use_command = false;
+                    use_function = false;
+                    use_builtin = true;
+                    break;
+            }
+
+            if (cmd_node && cmd_node->location_in_or_at_end_of_source_range(pos))
+            {
+                /* Complete command filename */
+                completer.complete_cmd(current_token, use_function, use_builtin, use_command);
+            }
+            else
+            {
+                /* Get all the arguments */
+                const parse_node_tree_t::parse_node_list_t all_arguments = tree.find_nodes(*plain_statement, symbol_argument);
+
+                /* See whether we are in an argument. We may also be in a redirection, or nothing at all. */
+                size_t matching_arg_index = -1;
+                for (size_t i=0; i < all_arguments.size(); i++)
                 {
-
-                    const wcstring ncmd = tok_last(&tok);
-                    int is_ddash = (ncmd == L"--") && ((tok_get_pos(&tok)+2) < (long)pos);
-
-                    if (!had_cmd)
+                    const parse_node_t *node = all_arguments.at(i);
+                    if (node->location_in_or_at_end_of_source_range(pos))
                     {
+                        matching_arg_index = i;
+                        break;
+                    }
+                }
 
-                        if (parser_keywords_is_subcommand(ncmd))
+                bool had_ddash = false;
+                wcstring current_argument, previous_argument;
+                if (matching_arg_index != (size_t)(-1))
+                {
+                    /* Get the current argument and the previous argument, if we have one */
+                    current_argument = all_arguments.at(matching_arg_index)->get_source(cmd);
+
+                    if (matching_arg_index > 0)
+                        previous_argument = all_arguments.at(matching_arg_index - 1)->get_source(cmd);
+
+                    /* Check to see if we have a preceding double-dash */
+                    for (size_t i=0; i < matching_arg_index; i++)
+                    {
+                        if (all_arguments.at(i)->get_source(cmd) == L"--")
                         {
-                            if (ncmd == L"builtin")
-                            {
-                                use_function = 0;
-                                use_command  = 0;
-                                use_builtin  = 1;
-                            }
-                            else if (ncmd == L"command")
-                            {
-                                use_command  = 1;
-                                use_function = 0;
-                                use_builtin  = 0;
-                            }
+                            had_ddash = true;
                             break;
                         }
-
-
-                        if (!is_ddash ||
-                                ((use_command && use_function && use_builtin)))
-                        {
-                            current_command = ncmd;
-
-                            size_t token_end = tok_get_pos(&tok) + ncmd.size();
-
-                            on_command = (pos <= token_end);
-                            had_cmd=1;
-                        }
-
                     }
-                    else
-                    {
-                        if (is_ddash)
-                        {
-                            had_ddash = 1;
-                        }
-                    }
-
-                    break;
                 }
 
-                case TOK_END:
-                case TOK_PIPE:
-                case TOK_BACKGROUND:
+                bool do_file = false;
+
+                wcstring current_command_unescape, previous_argument_unescape, current_argument_unescape;
+                if (unescape_string(current_command, &current_command_unescape, UNESCAPE_DEFAULT) &&
+                        unescape_string(previous_argument, &previous_argument_unescape, UNESCAPE_DEFAULT) &&
+                        unescape_string(current_argument, &current_argument_unescape, UNESCAPE_INCOMPLETE))
                 {
-                    had_cmd=0;
-                    had_ddash = 0;
-                    use_command  = 1;
-                    use_function = 1;
-                    use_builtin  = 1;
-                    break;
+                    do_file = completer.complete_param(current_command_unescape,
+                                                       previous_argument_unescape,
+                                                       current_argument_unescape,
+                                                       !had_ddash);
                 }
 
-                case TOK_ERROR:
-                {
-                    end_loop=1;
-                    break;
-                }
+                /* If we have found no command specific completions at all, fall back to using file completions. */
+                if (completer.empty())
+                    do_file = true;
 
-                default:
-                {
-                    break;
-                }
+                /* And if we're autosuggesting, and the token is empty, don't do file suggestions */
+                if ((flags & COMPLETION_REQUEST_AUTOSUGGESTION) && current_argument_unescape.empty())
+                    do_file = false;
+
+                /* This function wants the unescaped string */
+                completer.complete_param_expand(current_token, do_file);
             }
-
-            if (tok_get_pos(&tok) >= (long)pos)
-            {
-                end_loop=1;
-            }
-
-            tok_next(&tok);
-
-        }
-
-        /*
-          Get the string to complete
-        */
-
-        current_token.assign(tok_begin, cursor_pos-(tok_begin-cmd_cstr));
-
-        if (prev_begin)
-        {
-            prev_token.assign(prev_begin, prev_end - prev_begin);
-        }
-        else
-        {
-            prev_token.clear();
-        }
-
-//    debug( 0, L"on_command: %d, %ls %ls\n", on_command, current_command, current_token );
-
-        /*
-          Check if we are using the 'command' or 'builtin' builtins
-          _and_ we are writing a switch instead of a command. In that
-          case, complete using the builtins completions, not using a
-          subcommand.
-        */
-
-        if ((on_command || current_token == L"--") &&
-                string_prefixes_string(L"-", current_token) &&
-                !(use_command && use_function && use_builtin))
-        {
-            if (use_command == 0)
-                current_command = L"builtin";
-            else
-                current_command = L"command";
-
-            had_cmd = 1;
-            on_command = 0;
-        }
-
-        /*
-          Use command completions if in between commands
-        */
-        if (!had_cmd)
-        {
-            on_command=1;
-        }
-
-
-        if (on_command)
-        {
-            /* Complete command filename */
-            completer.complete_cmd(current_token, use_function, use_builtin, use_command);
-        }
-        else
-        {
-            bool do_file = false;
-
-            wcstring current_command_unescape, prev_token_unescape, current_token_unescape;
-            if (unescape_string(current_command, &current_command_unescape, UNESCAPE_DEFAULT) &&
-                unescape_string(prev_token, &prev_token_unescape, UNESCAPE_DEFAULT) &&
-                unescape_string(current_token, &current_token_unescape, UNESCAPE_INCOMPLETE))
-            {
-                do_file = completer.complete_param(current_command_unescape,
-                                                   prev_token_unescape,
-                                                   current_token_unescape,
-                                                   !had_ddash);
-            }
-
-            /* If we have found no command specific completions at
-            all, fall back to using file completions.
-            */
-            if (completer.empty())
-                do_file = true;
-
-            /* If we're autosuggesting, and the token is empty, don't do file suggestions */
-            if ((flags & COMPLETION_REQUEST_AUTOSUGGESTION) && current_token_unescape.empty())
-                do_file = false;
-
-            /*
-              This function wants the unescaped string
-            */
-            completer.complete_param_expand(current_token, do_file);
         }
     }
 

complete.h

@@ -121,10 +121,10 @@ public:
        The COMPLETE_NO_CASE can be used to signal that this completion
        is case insensitive.
     */
-    int flags;
+    complete_flags_t flags;
 
     /* Construction. Note: defining these so that they are not inlined reduces the executable size. */
-    completion_t(const wcstring &comp, const wcstring &desc = L"", string_fuzzy_match_t match = string_fuzzy_match_t(fuzzy_match_exact), int flags_val = 0);
+    completion_t(const wcstring &comp, const wcstring &desc = wcstring(), string_fuzzy_match_t match = string_fuzzy_match_t(fuzzy_match_exact), complete_flags_t flags_val = 0);
     completion_t(const completion_t &);
     completion_t &operator=(const completion_t &);
 
@@ -268,7 +268,7 @@ void complete_load(const wcstring &cmd, bool reload);
    \param flags completion flags
 
 */
-void append_completion(std::vector<completion_t> &completions, const wcstring &comp, const wcstring &desc = L"", int flags = 0, string_fuzzy_match_t match = string_fuzzy_match_t(fuzzy_match_exact));
+void append_completion(std::vector<completion_t> &completions, const wcstring &comp, const wcstring &desc = wcstring(), int flags = 0, string_fuzzy_match_t match = string_fuzzy_match_t(fuzzy_match_exact));
 
 /* Function used for testing */
 void complete_set_variable_names(const wcstring_list_t *names);

doc_src/tutorial.hdr

@@ -312,7 +312,7 @@ lesson.pdf
 /var/run/sntp.log
 </pre>
 
-<p>If that directory traversal is taking a long time, you can control-C out of it.
+<p>If that directory traversal is taking a long time, you can Control-C out of it.
 
 <h2 id="tut_pipes_and_redirections">Pipes and Redirections</h2>
 
@@ -500,7 +500,7 @@ You can iterate over a list (or a slice) with a <i>for loop</i>:
 > <b>for</b> <i>val</i> <b>in</b> <i>$PATH</i>
         <b>echo</b> <i>"entry: $val"</i>
   <b>end</b>
-entry: usr/bin/
+entry: /usr/bin/
 entry: /bin
 entry: /usr/sbin
 entry: /sbin

env.cpp

@@ -299,7 +299,6 @@ static bool var_is_locale(const wcstring &key)
 static void handle_locale()
 {
     const env_var_t lc_all = env_get_string(L"LC_ALL");
-    int i;
     const wcstring old_locale = wsetlocale(LC_MESSAGES, NULL);
 
     /*
@@ -330,7 +329,7 @@ static void handle_locale()
             wsetlocale(LC_ALL, lang.c_str());
         }
 
-        for (i=2; locale_variable[i]; i++)
+        for (int i=2; locale_variable[i]; i++)
         {
             const env_var_t val = env_get_string(locale_variable[i]);
 
@@ -479,7 +478,7 @@ static void env_set_defaults()
         if (pw->pw_name != NULL)
         {
             const wcstring wide_name = str2wcstring(pw->pw_name);
-            env_set(L"USER", NULL, ENV_GLOBAL);
+            env_set(L"USER", wide_name.c_str(), ENV_GLOBAL);
         }
     }
 
@@ -892,6 +891,7 @@ int env_set(const wcstring &key, const wchar_t *val, int var_mode)
     if (!is_universal)
     {
         event_t ev = event_t::variable_event(key);
+        ev.arguments.reserve(3);
         ev.arguments.push_back(L"VARIABLE");
         ev.arguments.push_back(L"SET");
         ev.arguments.push_back(key);

env_universal.cpp

@@ -132,12 +132,12 @@ static int try_get_socket_once(void)
     if (connect(s, (struct sockaddr *)&local, sizeof local) == -1)
     {
         close(s);
-        
+
         /* If it fails on first try, it's probably no serious error, but fishd hasn't been launched yet.
          This happens (at least) on the first concurrent session. */
         if (get_socket_count > 1)
             wperror(L"connect");
-        
+
         return -1;
     }
 
@@ -426,8 +426,6 @@ void env_universal_barrier()
 
 void env_universal_set(const wcstring &name, const wcstring &value, bool exportv)
 {
-    message_t *msg;
-
     if (!s_env_univeral_inited)
         return;
 
@@ -439,9 +437,9 @@ void env_universal_set(const wcstring &name, const wcstring &value, bool exportv
     }
     else
     {
-        msg = create_message(exportv?SET_EXPORT:SET,
-                             name.c_str(),
-                             value.c_str());
+        message_t *msg = create_message(exportv?SET_EXPORT:SET,
+                                        name.c_str(),
+                                        value.c_str());
 
         if (!msg)
         {
@@ -459,7 +457,6 @@ int env_universal_remove(const wchar_t *name)
 {
     int res;
 
-    message_t *msg;
     if (!s_env_univeral_inited)
         return 1;
 
@@ -476,7 +473,7 @@ int env_universal_remove(const wchar_t *name)
     }
     else
     {
-        msg= create_message(ERASE, name, 0);
+        message_t *msg = create_message(ERASE, name, 0);
         msg->count=1;
         env_universal_server.unsent.push(msg);
         env_universal_barrier();

event.cpp

@@ -146,13 +146,13 @@ static int event_is_blocked(const event_t &e)
 {
     const block_t *block;
     parser_t &parser = parser_t::principal_parser();
-    
+
     size_t idx = 0;
     while ((block = parser.block_at_index(idx++)))
     {
         if (event_block_list_blocks_type(block->event_blocks, e.type))
             return true;
-        
+
     }
     return event_block_list_blocks_type(parser.global_event_blocks, e.type);
 }
@@ -568,9 +568,6 @@ static void event_fire_internal(const event_t &event)
 */
 static void event_fire_delayed()
 {
-
-    size_t i;
-
     /*
       If is_event is one, we are running the event-handler non-recursively.
 
@@ -582,7 +579,7 @@ static void event_fire_delayed()
     {
         event_list_t new_blocked;
 
-        for (i=0; i<blocked.size(); i++)
+        for (size_t i=0; i<blocked.size(); i++)
         {
             event_t *e = blocked.at(i);
             if (event_is_blocked(*e))

exec.cpp

@@ -394,12 +394,13 @@ static void io_cleanup_fds(const std::vector<int> &opened_fds)
    repeatedly reopened for every command in the block, which would
    reset the cursor position.
 
-   \return the transmogrified chain on sucess, or 0 on failiure
+   \return true on success, false on failure. Returns the output chain and opened_fds by reference
 */
-static bool io_transmogrify(const io_chain_t &in_chain, io_chain_t &out_chain, std::vector<int> &out_opened_fds)
+static bool io_transmogrify(const io_chain_t &in_chain, io_chain_t *out_chain, std::vector<int> *out_opened_fds)
 {
     ASSERT_IS_MAIN_THREAD();
-    assert(out_chain.empty());
+    assert(out_chain != NULL && out_opened_fds != NULL);
+    assert(out_chain->empty());
 
     /* Just to be clear what we do for an empty chain */
     if (in_chain.empty())
@@ -479,8 +480,8 @@ static bool io_transmogrify(const io_chain_t &in_chain, io_chain_t &out_chain, s
     if (success)
     {
         /* Yay */
-        out_chain.swap(result_chain);
-        out_opened_fds.swap(opened_fds);
+        out_chain->swap(result_chain);
+        out_opened_fds->swap(opened_fds);
     }
     else
     {
@@ -496,19 +497,24 @@ static bool io_transmogrify(const io_chain_t &in_chain, io_chain_t &out_chain, s
    Morph an io redirection chain into redirections suitable for
    passing to eval, call eval, and clean up morphed redirections.
 
-   \param def the code to evaluate
+   \param def the code to evaluate, or the empty string if none
+   \param node_offset the offset of the node to evalute, or NODE_OFFSET_INVALID
    \param block_type the type of block to push on evaluation
    \param io the io redirections to be performed on this block
 */
 
 static void internal_exec_helper(parser_t &parser,
-                                 const wchar_t *def,
+                                 const wcstring &def,
+                                 node_offset_t node_offset,
                                  enum block_type_t block_type,
                                  const io_chain_t &ios)
 {
+    // If we have a valid node offset, then we must not have a string to execute
+    assert(node_offset == NODE_OFFSET_INVALID || def.empty());
+
     io_chain_t morphed_chain;
     std::vector<int> opened_fds;
-    bool transmorgrified = io_transmogrify(ios, morphed_chain, opened_fds);
+    bool transmorgrified = io_transmogrify(ios, &morphed_chain, &opened_fds);
 
     int is_block_old=is_block;
     is_block=1;
@@ -524,7 +530,14 @@ static void internal_exec_helper(parser_t &parser,
 
     signal_unblock();
 
-    parser.eval(def, morphed_chain, block_type);
+    if (node_offset == NODE_OFFSET_INVALID)
+    {
+        parser.eval(def, morphed_chain, block_type);
+    }
+    else
+    {
+        parser.eval_block_node(node_offset, morphed_chain, block_type);
+    }
 
     signal_block();
 
@@ -564,6 +577,12 @@ static bool can_use_posix_spawn_for_job(const job_t *job, const process_t *proce
 /* What exec does if no_exec is set. This only has to handle block pushing and popping. See #624. */
 static void exec_no_exec(parser_t &parser, const job_t *job)
 {
+    if (parser_use_ast())
+    {
+        /* With the new parser, commands aren't responsible for pushing / popping blocks, so there's nothing to do */
+        return;
+    }
+
     /* Hack hack hack. If this is an 'end' job, then trigger a pop. If this is a job that would create a block, trigger a push. See #624 */
     const process_t *p = job->first_process;
     if (p && p->type == INTERNAL_BUILTIN)
@@ -682,7 +701,7 @@ void exec_job(parser_t &parser, job_t *j)
             j->first_process->completed=1;
             return;
         }
-
+        assert(0 && "This should be unreachable");
     }
 
     signal_block();
@@ -807,7 +826,6 @@ void exec_job(parser_t &parser, job_t *j)
         {
             pipe_write.reset(new io_pipe_t(p->pipe_write_fd, false));
             process_net_io_chain.push_back(pipe_write);
-
         }
 
         /* The explicit IO redirections associated with the process */
@@ -926,7 +944,7 @@ void exec_job(parser_t &parser, job_t *j)
 
                 if (! exec_error)
                 {
-                    internal_exec_helper(parser, def.c_str(), TOP, process_net_io_chain);
+                    internal_exec_helper(parser, def, NODE_OFFSET_INVALID, TOP, process_net_io_chain);
                 }
 
                 parser.allow_function();
@@ -936,12 +954,14 @@ void exec_job(parser_t &parser, job_t *j)
             }
 
             case INTERNAL_BLOCK:
+            case INTERNAL_BLOCK_NODE:
             {
                 if (p->next)
                 {
                     block_output_io_buffer.reset(io_buffer_t::create(0));
                     if (block_output_io_buffer.get() == NULL)
                     {
+                        /* We failed (e.g. no more fds could be created). */
                         exec_error = true;
                         job_mark_process_as_failed(j, p);
                     }
@@ -954,10 +974,19 @@ void exec_job(parser_t &parser, job_t *j)
 
                 if (! exec_error)
                 {
-                    internal_exec_helper(parser, p->argv0(), TOP, process_net_io_chain);
+                    if (p->type == INTERNAL_BLOCK)
+                    {
+                        /* The block contents (as in, fish code) are stored in argv0 (ugh) */
+                        assert(p->argv0() != NULL);
+                        internal_exec_helper(parser, p->argv0(), NODE_OFFSET_INVALID, TOP, process_net_io_chain);
+                    }
+                    else
+                    {
+                        assert(p->type == INTERNAL_BLOCK_NODE);
+                        internal_exec_helper(parser, wcstring(), p->internal_block_node, TOP, process_net_io_chain);
+                    }
                 }
                 break;
-
             }
 
             case INTERNAL_BUILTIN:
@@ -1104,6 +1133,20 @@ void exec_job(parser_t &parser, job_t *j)
                 }
                 break;
             }
+
+            case EXTERNAL:
+                /* External commands are handled in the next switch statement below */
+                break;
+
+            case INTERNAL_EXEC:
+                /* We should have handled exec up above */
+                assert(0 && "INTERNAL_EXEC process found in pipeline, where it should never be. Aborting.");
+                break;
+
+            case INTERNAL_BUFFER:
+                /* Internal buffers are handled in the next switch statement below */
+                break;
+
         }
 
         if (exec_error)
@@ -1115,6 +1158,7 @@ void exec_job(parser_t &parser, job_t *j)
         {
 
             case INTERNAL_BLOCK:
+            case INTERNAL_BLOCK_NODE:
             case INTERNAL_FUNCTION:
             {
                 int status = proc_get_last_status();
@@ -1131,7 +1175,7 @@ void exec_job(parser_t &parser, job_t *j)
                       No buffer, so we exit directly. This means we
                       have to manually set the exit status.
                     */
-                    if (p->next == 0)
+                    if (p->next == NULL)
                     {
                         proc_set_last_status(job_get_flag(j, JOB_NEGATE)?(!status):status);
                     }
@@ -1464,6 +1508,12 @@ void exec_job(parser_t &parser, job_t *j)
                 break;
             }
 
+            case INTERNAL_EXEC:
+            {
+                /* We should have handled exec up above */
+                assert(0 && "INTERNAL_EXEC process found in pipeline, where it should never be. Aborting.");
+                break;
+            }
         }
 
         if (p->type == INTERNAL_BUILTIN)
@@ -1532,6 +1582,8 @@ static int exec_subshell_internal(const wcstring &cmd, wcstring_list_t *lst, boo
     const int prev_status = proc_get_last_status();
     char sep=0;
 
+    //fprintf(stderr, "subcmd %ls\n", cmd.c_str());
+
     const env_var_t ifs = env_get_string(L"IFS");
 
     if (! ifs.missing_or_empty())

expand.cpp

@@ -785,6 +785,14 @@ static int expand_pid(const wcstring &instr_with_sep,
                       expand_flags_t flags,
                       std::vector<completion_t> &out)
 {
+    /* Hack. If there's no INTERNAL_SEP and no PROCESS_EXPAND, then there's nothing to do. Check out this "null terminated string." */
+    const wchar_t some_chars[] = {INTERNAL_SEPARATOR, PROCESS_EXPAND, L'\0'};
+    if (instr_with_sep.find_first_of(some_chars) == wcstring::npos)
+    {
+        /* Nothing to do */
+        append_completion(out, instr_with_sep);
+        return 1;
+    }
 
     /* expand_string calls us with internal separators in instr...sigh */
     wcstring instr = instr_with_sep;
@@ -1372,7 +1380,7 @@ static int expand_brackets(parser_t &parser, const wcstring &instr, int flags, s
 /**
  Perform cmdsubst expansion
  */
-static int expand_cmdsubst(parser_t &parser, const wcstring &input, std::vector<completion_t> &outList)
+static int expand_cmdsubst(parser_t &parser, const wcstring &input, std::vector<completion_t> &out_list)
 {
     wchar_t *paran_begin=0, *paran_end=0;
     std::vector<wcstring> sub_res;
@@ -1390,7 +1398,7 @@ static int expand_cmdsubst(parser_t &parser, const wcstring &input, std::vector<
                          L"Mismatched parenthesis");
             return 0;
         case 0:
-            outList.push_back(completion_t(input));
+            append_completion(out_list, input);
             return 1;
         case 1:
 
@@ -1455,15 +1463,15 @@ static int expand_cmdsubst(parser_t &parser, const wcstring &input, std::vector<
        */
     for (i=0; i<sub_res.size(); i++)
     {
-        wcstring sub_item = sub_res.at(i);
-        wcstring sub_item2 = escape_string(sub_item, 1);
+        const wcstring &sub_item = sub_res.at(i);
+        const wcstring sub_item2 = escape_string(sub_item, 1);
+
+        wcstring whole_item;
 
         for (j=0; j < tail_expand.size(); j++)
         {
-
-            wcstring whole_item;
-
-            wcstring tail_item = tail_expand.at(j).completion;
+            whole_item.clear();
+            const wcstring &tail_item = tail_expand.at(j).completion;
 
             //sb_append_substring( &whole_item, in, len1 );
             whole_item.append(in, paran_begin-in);
@@ -1481,7 +1489,7 @@ static int expand_cmdsubst(parser_t &parser, const wcstring &input, std::vector<
             whole_item.append(tail_item);
 
             //al_push( out, whole_item.buff );
-            outList.push_back(completion_t(whole_item));
+            append_completion(out_list, whole_item);
         }
     }
 
@@ -1602,6 +1610,31 @@ static void unexpand_tildes(const wcstring &input, std::vector<completion_t> *co
     }
 }
 
+// If the given path contains the user's home directory, replace that with a tilde
+// We don't try to be smart about case insensitivity, etc.
+wcstring replace_home_directory_with_tilde(const wcstring &str)
+{
+    // only absolute paths get this treatment
+    wcstring result = str;
+    if (string_prefixes_string(L"/", result))
+    {
+        wcstring home_directory = L"~";
+        expand_tilde(home_directory);
+        if (! string_suffixes_string(L"/", home_directory))
+        {
+            home_directory.push_back(L'/');
+        }
+
+        // Now check if the home_directory prefixes the string
+        if (string_prefixes_string(home_directory, result))
+        {
+            // Success
+            result.replace(0, home_directory.size(), L"~/");
+        }
+    }
+    return result;
+}
+
 /**
    Remove any internal separators. Also optionally convert wildcard characters to
    regular equivalents. This is done to support EXPAND_SKIP_WILDCARDS.
@@ -1640,7 +1673,7 @@ int expand_string(const wcstring &input, std::vector<completion_t> &output, expa
 
     if ((!(flags & ACCEPT_INCOMPLETE)) && expand_is_clean(input.c_str()))
     {
-        output.push_back(completion_t(input));
+        append_completion(output, input);
         return EXPAND_OK;
     }
 
@@ -1656,7 +1689,7 @@ int expand_string(const wcstring &input, std::vector<completion_t> &output, expa
             parser.error(CMDSUBST_ERROR, -1, L"Command substitutions not allowed");
             return EXPAND_ERROR;
         }
-        in->push_back(completion_t(input));
+        append_completion(*in, input);
     }
     else
     {
@@ -1684,7 +1717,7 @@ int expand_string(const wcstring &input, std::vector<completion_t> &output, expa
                     next[i] = L'$';
                 }
             }
-            out->push_back(completion_t(next));
+            append_completion(*out, next);
         }
         else
         {
@@ -1700,7 +1733,7 @@ int expand_string(const wcstring &input, std::vector<completion_t> &output, expa
 
     for (i=0; i < in->size(); i++)
     {
-        wcstring next = in->at(i).completion;
+        const wcstring &next = in->at(i).completion;
 
         if (!expand_brackets(parser, next, flags, *out))
         {
@@ -1720,7 +1753,7 @@ int expand_string(const wcstring &input, std::vector<completion_t> &output, expa
 
         if (flags & ACCEPT_INCOMPLETE)
         {
-            if (next[0] == PROCESS_EXPAND)
+            if (! next.empty() && next.at(0) == PROCESS_EXPAND)
             {
                 /*
                  If process expansion matches, we are not
@@ -1733,7 +1766,7 @@ int expand_string(const wcstring &input, std::vector<completion_t> &output, expa
             }
             else
             {
-                out->push_back(completion_t(next));
+                append_completion(*out, next);
             }
         }
         else
@@ -1815,7 +1848,7 @@ int expand_string(const wcstring &input, std::vector<completion_t> &output, expa
         {
             if (!(flags & ACCEPT_INCOMPLETE))
             {
-                out->push_back(completion_t(next_str));
+                append_completion(*out, next_str);
             }
         }
     }
@@ -1842,7 +1875,7 @@ bool expand_one(wcstring &string, expand_flags_t flags)
         return true;
     }
 
-    if (expand_string(string, completions, flags))
+    if (expand_string(string, completions, flags | EXPAND_NO_DESCRIPTIONS))
     {
         if (completions.size() == 1)
         {
@@ -1945,19 +1978,19 @@ bool fish_openSUSE_dbus_hack_hack_hack_hack(std::vector<completion_t> *args)
                         val.resize(last_good + 1);
 
                     args->clear();
-                    args->push_back(completion_t(L"set"));
+                    append_completion(*args, L"set");
                     if (key == L"DBUS_SESSION_BUS_ADDRESS")
-                        args->push_back(completion_t(L"-x"));
-                    args->push_back(completion_t(key));
-                    args->push_back(completion_t(val));
+                        append_completion(*args, L"-x");
+                    append_completion(*args, key);
+                    append_completion(*args, val);
                     result = true;
                 }
                 else if (string_prefixes_string(L"export DBUS_SESSION_BUS_ADDRESS;", cmd))
                 {
                     /* Nothing, we already exported it */
                     args->clear();
-                    args->push_back(completion_t(L"echo"));
-                    args->push_back(completion_t(L"-n"));
+                    append_completion(*args, L"echo");
+                    append_completion(*args, L"-n");
                     result = true;
                 }
             }

expand.h

@@ -176,6 +176,9 @@ wcstring expand_escape_variable(const wcstring &in);
 */
 void expand_tilde(wcstring &input);
 
+/** Perform the opposite of tilde expansion on the string, which is modified in place */
+wcstring replace_home_directory_with_tilde(const wcstring &str);
+
 /**
    Test if the specified argument is clean, i.e. it does not contain
    any tokens which need to be expanded or otherwise altered. Clean

fish.xcodeproj/project.pbxproj

@@ -65,6 +65,7 @@
 		D033781115DC6D4C00A634BA /* completions in CopyFiles */ = {isa = PBXBuildFile; fileRef = D025C02715D1FEA100B9DB63 /* completions */; };
 		D033781215DC6D5200A634BA /* functions in CopyFiles */ = {isa = PBXBuildFile; fileRef = D025C02815D1FEA100B9DB63 /* functions */; };
 		D033781315DC6D5400A634BA /* tools in CopyFiles */ = {isa = PBXBuildFile; fileRef = D025C02915D1FEA100B9DB63 /* tools */; };
+		D052D80B1868F7FC003ABCBD /* parse_execution.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D052D8091868F7FC003ABCBD /* parse_execution.cpp */; };
 		D07B247315BCC15700D4ADB4 /* add-shell in Resources */ = {isa = PBXBuildFile; fileRef = D07B247215BCC15700D4ADB4 /* add-shell */; };
 		D07B247615BCC4BE00D4ADB4 /* install.sh in Resources */ = {isa = PBXBuildFile; fileRef = D07B247515BCC4BE00D4ADB4 /* install.sh */; };
 		D07D266A15E33B86009E43F6 /* config.fish in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0C4FD9415A7D7EE00212EF1 /* config.fish */; };
@@ -73,8 +74,50 @@
 		D07D266E15E33B86009E43F6 /* tools in Copy Files */ = {isa = PBXBuildFile; fileRef = D025C02915D1FEA100B9DB63 /* tools */; };
 		D07D267215E34171009E43F6 /* config.fish in Copy Files */ = {isa = PBXBuildFile; fileRef = D0CBD580159EE48F0024809C /* config.fish */; };
 		D0879AC816BF9AAB00E98E56 /* fish_term_icon.icns in Resources */ = {isa = PBXBuildFile; fileRef = D0879AC616BF9A1A00E98E56 /* fish_term_icon.icns */; };
+		D08A329417B4458D00F3A533 /* fish_tests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D08A329317B4458D00F3A533 /* fish_tests.cpp */; };
+		D08A329517B445C200F3A533 /* function.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854413B3ACEE0099B651 /* function.cpp */; };
+		D08A329617B445FD00F3A533 /* builtin.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853513B3ACEE0099B651 /* builtin.cpp */; };
+		D08A329717B4463B00F3A533 /* complete.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853713B3ACEE0099B651 /* complete.cpp */; };
+		D08A329817B4463B00F3A533 /* env.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853A13B3ACEE0099B651 /* env.cpp */; };
+		D08A329917B4463B00F3A533 /* exec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853C13B3ACEE0099B651 /* exec.cpp */; };
+		D08A329A17B4463B00F3A533 /* expand.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853D13B3ACEE0099B651 /* expand.cpp */; };
+		D08A329B17B4463B00F3A533 /* highlight.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854713B3ACEE0099B651 /* highlight.cpp */; };
+		D08A329C17B4463B00F3A533 /* history.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854813B3ACEE0099B651 /* history.cpp */; };
+		D08A329D17B4463B00F3A533 /* kill.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854F13B3ACEE0099B651 /* kill.cpp */; };
+		D08A329E17B4463B00F3A533 /* parser.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855413B3ACEE0099B651 /* parser.cpp */; };
+		D08A329F17B4463B00F3A533 /* proc.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855713B3ACEE0099B651 /* proc.cpp */; };
+		D08A32A017B4463B00F3A533 /* reader.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855813B3ACEE0099B651 /* reader.cpp */; };
+		D08A32A117B4463B00F3A533 /* sanity.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855913B3ACEE0099B651 /* sanity.cpp */; };
+		D08A32A217B4463B00F3A533 /* tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855D13B3ACEE0099B651 /* tokenizer.cpp */; };
+		D08A32A317B4463B00F3A533 /* wgetopt.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855F13B3ACEE0099B651 /* wgetopt.cpp */; };
+		D08A32A417B4463B00F3A533 /* wildcard.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0856013B3ACEE0099B651 /* wildcard.cpp */; };
+		D08A32A517B4463B00F3A533 /* wutil.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0856113B3ACEE0099B651 /* wutil.cpp */; };
+		D08A32A617B4464300F3A533 /* input.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854A13B3ACEE0099B651 /* input.cpp */; };
+		D08A32A717B446A300F3A533 /* autoload.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C6FCC914CFA4B0004CE8AD /* autoload.cpp */; };
+		D08A32A817B446A300F3A533 /* builtin_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0F3373A1506DE3C00ECEFC0 /* builtin_test.cpp */; };
+		D08A32A917B446A300F3A533 /* color.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0B6B0FE14E88BA400AD6C10 /* color.cpp */; };
+		D08A32AA17B446A300F3A533 /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; };
+		D08A32AB17B446A300F3A533 /* env_universal_common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853813B3ACEE0099B651 /* env_universal_common.cpp */; };
+		D08A32AC17B446A300F3A533 /* env_universal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853913B3ACEE0099B651 /* env_universal.cpp */; };
+		D08A32AD17B446A300F3A533 /* event.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853B13B3ACEE0099B651 /* event.cpp */; };
+		D08A32AE17B446A300F3A533 /* input_common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854913B3ACEE0099B651 /* input_common.cpp */; };
+		D08A32AF17B446A300F3A533 /* intern.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854B13B3ACEE0099B651 /* intern.cpp */; };
+		D08A32B017B446A300F3A533 /* io.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854C13B3ACEE0099B651 /* io.cpp */; };
+		D08A32B117B446A300F3A533 /* iothread.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854D13B3ACEE0099B651 /* iothread.cpp */; };
+		D08A32B217B446A300F3A533 /* output.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855113B3ACEE0099B651 /* output.cpp */; };
+		D08A32B317B446A300F3A533 /* parse_util.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855213B3ACEE0099B651 /* parse_util.cpp */; };
+		D08A32B417B446A300F3A533 /* parser_keywords.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855313B3ACEE0099B651 /* parser_keywords.cpp */; };
+		D08A32B517B446A300F3A533 /* path.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855513B3ACEE0099B651 /* path.cpp */; };
+		D08A32B617B446A300F3A533 /* postfork.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D09B1C1914FC7B5B00F91077 /* postfork.cpp */; };
+		D08A32B717B446A300F3A533 /* screen.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855A13B3ACEE0099B651 /* screen.cpp */; };
+		D08A32B817B446A300F3A533 /* signal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855C13B3ACEE0099B651 /* signal.cpp */; };
+		D08A32B917B446B100F3A533 /* parse_productions.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */; };
+		D08A32BA17B446B100F3A533 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; };
+		D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
+		D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
 		D0A564FE168D23D800AF6161 /* man in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; };
 		D0A56501168D258300AF6161 /* man in Copy Files */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; };
+		D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; };
 		D0CBD587159EF0E10024809C /* launch_fish.scpt in Resources */ = {isa = PBXBuildFile; fileRef = D0CBD586159EF0E10024809C /* launch_fish.scpt */; };
 		D0D02A67159837AD008E62BD /* complete.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853713B3ACEE0099B651 /* complete.cpp */; };
 		D0D02A69159837B2008E62BD /* env.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853A13B3ACEE0099B651 /* env.cpp */; };
@@ -150,6 +193,7 @@
 		D0F019FD15A977CA0034B3B1 /* config.fish in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0C4FD9415A7D7EE00212EF1 /* config.fish */; };
 		D0F01A0315A978910034B3B1 /* osx_fish_launcher.m in Sources */ = {isa = PBXBuildFile; fileRef = D0D02AFA159871B2008E62BD /* osx_fish_launcher.m */; };
 		D0F01A0515A978A10034B3B1 /* Foundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0CBD583159EEE010024809C /* Foundation.framework */; };
+		D0FE8EE8179FB760008C9F21 /* parse_productions.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -292,6 +336,15 @@
 			name = "Copy Files";
 			runOnlyForDeploymentPostprocessing = 1;
 		};
+		D08A328B17B4455100F3A533 /* CopyFiles */ = {
+			isa = PBXCopyFilesBuildPhase;
+			buildActionMask = 2147483647;
+			dstPath = /usr/share/man/man1/;
+			dstSubfolderSpec = 0;
+			files = (
+			);
+			runOnlyForDeploymentPostprocessing = 1;
+		};
 		D0F019F015A977010034B3B1 /* CopyFiles */ = {
 			isa = PBXCopyFilesBuildPhase;
 			buildActionMask = 2147483647;
@@ -334,9 +387,13 @@
 		D025C02915D1FEA100B9DB63 /* tools */ = {isa = PBXFileReference; lastKnownFileType = folder; name = tools; path = share/tools; sourceTree = "<group>"; };
 		D031890915E36D9800D9CC39 /* base */ = {isa = PBXFileReference; lastKnownFileType = text; path = base; sourceTree = BUILT_PRODUCTS_DIR; };
 		D03EE83814DF88B200FC7150 /* lru.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = lru.h; sourceTree = "<group>"; };
+		D052D8091868F7FC003ABCBD /* parse_execution.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_execution.cpp; sourceTree = "<group>"; };
+		D052D80A1868F7FC003ABCBD /* parse_execution.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_execution.h; sourceTree = "<group>"; };
 		D07B247215BCC15700D4ADB4 /* add-shell */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; name = "add-shell"; path = "build_tools/osx_package_scripts/add-shell"; sourceTree = "<group>"; };
 		D07B247515BCC4BE00D4ADB4 /* install.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; name = install.sh; path = osx/install.sh; sourceTree = "<group>"; };
 		D0879AC616BF9A1A00E98E56 /* fish_term_icon.icns */ = {isa = PBXFileReference; lastKnownFileType = image.icns; name = fish_term_icon.icns; path = osx/fish_term_icon.icns; sourceTree = "<group>"; };
+		D08A328D17B4455100F3A533 /* fish_tests */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fish_tests; sourceTree = BUILT_PRODUCTS_DIR; };
+		D08A329317B4458D00F3A533 /* fish_tests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = fish_tests.cpp; sourceTree = "<group>"; };
 		D09B1C1914FC7B5B00F91077 /* postfork.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = postfork.cpp; sourceTree = "<group>"; };
 		D09B1C1A14FC7B5B00F91077 /* postfork.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = postfork.h; sourceTree = "<group>"; };
 		D0A0850313B3ACEE0099B651 /* builtin.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = builtin.h; sourceTree = "<group>"; };
@@ -441,6 +498,8 @@
 		D0B6B0FE14E88BA400AD6C10 /* color.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = color.cpp; sourceTree = "<group>"; };
 		D0B6B0FF14E88BA400AD6C10 /* color.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = color.h; sourceTree = "<group>"; };
 		D0C4FD9415A7D7EE00212EF1 /* config.fish */ = {isa = PBXFileReference; lastKnownFileType = text; name = config.fish; path = etc/config.fish; sourceTree = "<group>"; };
+		D0C52F351765284C00BFAB82 /* parse_tree.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_tree.cpp; sourceTree = "<group>"; };
+		D0C52F361765284C00BFAB82 /* parse_tree.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_tree.h; sourceTree = "<group>"; };
 		D0C6FCC914CFA4B0004CE8AD /* autoload.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = autoload.cpp; sourceTree = "<group>"; };
 		D0C6FCCB14CFA4B7004CE8AD /* autoload.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = autoload.h; sourceTree = "<group>"; };
 		D0C861EA16CC7054003B5A04 /* builtin_set_color.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_set_color.cpp; sourceTree = "<group>"; };
@@ -457,11 +516,23 @@
 		D0D02AE415986537008E62BD /* fish_pager */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fish_pager; sourceTree = BUILT_PRODUCTS_DIR; };
 		D0D02AFA159871B2008E62BD /* osx_fish_launcher.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = osx_fish_launcher.m; path = osx/osx_fish_launcher.m; sourceTree = "<group>"; };
 		D0D2693C159835CA005D9B9C /* fish */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fish; sourceTree = BUILT_PRODUCTS_DIR; };
+		D0D9B2B318555D92001AE279 /* parse_constants.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = parse_constants.h; sourceTree = "<group>"; };
 		D0F3373A1506DE3C00ECEFC0 /* builtin_test.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_test.cpp; sourceTree = "<group>"; };
 		D0F5E28415A7A32D00315DFF /* config.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = config.h; sourceTree = "<group>"; };
+		D0FE8EE6179CA8A5008C9F21 /* parse_productions.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_productions.h; sourceTree = "<group>"; };
+		D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_productions.cpp; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
+		D08A328A17B4455100F3A533 /* Frameworks */ = {
+			isa = PBXFrameworksBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */,
+				D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
 		D0D02AB915985EF9008E62BD /* Frameworks */ = {
 			isa = PBXFrameworksBuildPhase;
 			buildActionMask = 2147483647;
@@ -525,6 +596,13 @@
 			name = "Other Build Products";
 			sourceTree = "<group>";
 		};
+		D08A328E17B4455100F3A533 /* fish_tests */ = {
+			isa = PBXGroup;
+			children = (
+			);
+			path = fish_tests;
+			sourceTree = "<group>";
+		};
 		D0A084F013B3AC130099B651 = {
 			isa = PBXGroup;
 			children = (
@@ -534,6 +612,7 @@
 				D0D02A8E15983D5F008E62BD /* Libraries */,
 				D0D02AAB15985C14008E62BD /* Resources */,
 				D031890A15E36DB500D9CC39 /* Other Build Products */,
+				D08A328E17B4455100F3A533 /* fish_tests */,
 				D0D2693215983562005D9B9C /* Products */,
 			);
 			sourceTree = "<group>";
@@ -582,6 +661,13 @@
 				D0A0853C13B3ACEE0099B651 /* exec.cpp */,
 				D0A0850C13B3ACEE0099B651 /* expand.h */,
 				D0A0853D13B3ACEE0099B651 /* expand.cpp */,
+				D0D9B2B318555D92001AE279 /* parse_constants.h */,
+				D0FE8EE6179CA8A5008C9F21 /* parse_productions.h */,
+				D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */,
+				D0C52F361765284C00BFAB82 /* parse_tree.h */,
+				D0C52F351765284C00BFAB82 /* parse_tree.cpp */,
+				D052D80A1868F7FC003ABCBD /* parse_execution.h */,
+				D052D8091868F7FC003ABCBD /* parse_execution.cpp */,
 				D0A0850D13B3ACEE0099B651 /* fallback.h */,
 				D0A0853E13B3ACEE0099B651 /* fallback.cpp */,
 				D0A0850E13B3ACEE0099B651 /* function.h */,
@@ -657,6 +743,7 @@
 				D0A0856613B3ACEE0099B651 /* xdgmimemagic.cpp */,
 				D0A0852F13B3ACEE0099B651 /* xdgmimeparent.h */,
 				D0A0856713B3ACEE0099B651 /* xdgmimeparent.cpp */,
+				D08A329317B4458D00F3A533 /* fish_tests.cpp */,
 			);
 			name = Sources;
 			sourceTree = "<group>";
@@ -698,6 +785,7 @@
 				D0D02ABC15985EF9008E62BD /* fishd */,
 				D0D02AD01598642A008E62BD /* fish_indent */,
 				D0D02AE415986537008E62BD /* fish_pager */,
+				D08A328D17B4455100F3A533 /* fish_tests */,
 			);
 			name = Products;
 			sourceTree = "<group>";
@@ -730,6 +818,23 @@
 /* End PBXLegacyTarget section */
 
 /* Begin PBXNativeTarget section */
+		D08A328C17B4455100F3A533 /* fish_tests */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = D08A329217B4455100F3A533 /* Build configuration list for PBXNativeTarget "fish_tests" */;
+			buildPhases = (
+				D08A328917B4455100F3A533 /* Sources */,
+				D08A328A17B4455100F3A533 /* Frameworks */,
+				D08A328B17B4455100F3A533 /* CopyFiles */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+			);
+			name = fish_tests;
+			productName = fish_tests;
+			productReference = D08A328D17B4455100F3A533 /* fish_tests */;
+			productType = "com.apple.product-type.tool";
+		};
 		D0D02A9915985A75008E62BD /* fish.app */ = {
 			isa = PBXNativeTarget;
 			buildConfigurationList = D0D02AA415985A75008E62BD /* Build configuration list for PBXNativeTarget "fish.app" */;
@@ -839,6 +944,7 @@
 				D0D02ABB15985EF9008E62BD /* fishd */,
 				D0D02ACF1598642A008E62BD /* fish_indent */,
 				D0D02AE315986537008E62BD /* fish_pager */,
+				D08A328C17B4455100F3A533 /* fish_tests */,
 				D0A564E6168CFDD800AF6161 /* man_pages */,
 				D0A084F713B3AC130099B651 /* Makefile */,
 			);
@@ -1019,6 +1125,52 @@
 /* End PBXShellScriptBuildPhase section */
 
 /* Begin PBXSourcesBuildPhase section */
+		D08A328917B4455100F3A533 /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				D08A32B917B446B100F3A533 /* parse_productions.cpp in Sources */,
+				D08A32BA17B446B100F3A533 /* parse_tree.cpp in Sources */,
+				D08A32A717B446A300F3A533 /* autoload.cpp in Sources */,
+				D08A32A817B446A300F3A533 /* builtin_test.cpp in Sources */,
+				D08A32A917B446A300F3A533 /* color.cpp in Sources */,
+				D08A32AA17B446A300F3A533 /* common.cpp in Sources */,
+				D08A32AB17B446A300F3A533 /* env_universal_common.cpp in Sources */,
+				D08A32AC17B446A300F3A533 /* env_universal.cpp in Sources */,
+				D08A32AD17B446A300F3A533 /* event.cpp in Sources */,
+				D08A32AE17B446A300F3A533 /* input_common.cpp in Sources */,
+				D08A32AF17B446A300F3A533 /* intern.cpp in Sources */,
+				D08A32B017B446A300F3A533 /* io.cpp in Sources */,
+				D08A32B117B446A300F3A533 /* iothread.cpp in Sources */,
+				D08A32B217B446A300F3A533 /* output.cpp in Sources */,
+				D08A32B317B446A300F3A533 /* parse_util.cpp in Sources */,
+				D08A32B417B446A300F3A533 /* parser_keywords.cpp in Sources */,
+				D08A32B517B446A300F3A533 /* path.cpp in Sources */,
+				D08A32B617B446A300F3A533 /* postfork.cpp in Sources */,
+				D08A32B717B446A300F3A533 /* screen.cpp in Sources */,
+				D08A32B817B446A300F3A533 /* signal.cpp in Sources */,
+				D08A32A617B4464300F3A533 /* input.cpp in Sources */,
+				D08A329717B4463B00F3A533 /* complete.cpp in Sources */,
+				D08A329817B4463B00F3A533 /* env.cpp in Sources */,
+				D08A329917B4463B00F3A533 /* exec.cpp in Sources */,
+				D08A329A17B4463B00F3A533 /* expand.cpp in Sources */,
+				D08A329B17B4463B00F3A533 /* highlight.cpp in Sources */,
+				D08A329C17B4463B00F3A533 /* history.cpp in Sources */,
+				D08A329D17B4463B00F3A533 /* kill.cpp in Sources */,
+				D08A329E17B4463B00F3A533 /* parser.cpp in Sources */,
+				D08A329F17B4463B00F3A533 /* proc.cpp in Sources */,
+				D08A32A017B4463B00F3A533 /* reader.cpp in Sources */,
+				D08A32A117B4463B00F3A533 /* sanity.cpp in Sources */,
+				D08A32A217B4463B00F3A533 /* tokenizer.cpp in Sources */,
+				D08A32A317B4463B00F3A533 /* wgetopt.cpp in Sources */,
+				D08A32A417B4463B00F3A533 /* wildcard.cpp in Sources */,
+				D08A32A517B4463B00F3A533 /* wutil.cpp in Sources */,
+				D08A329617B445FD00F3A533 /* builtin.cpp in Sources */,
+				D08A329417B4458D00F3A533 /* fish_tests.cpp in Sources */,
+				D08A329517B445C200F3A533 /* function.cpp in Sources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
 		D0D02AB815985EF9008E62BD /* Sources */ = {
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
@@ -1075,6 +1227,7 @@
 				D0D02A83159839D5008E62BD /* iothread.cpp in Sources */,
 				D0D02A84159839D5008E62BD /* parse_util.cpp in Sources */,
 				D0D02A85159839D5008E62BD /* path.cpp in Sources */,
+				D052D80B1868F7FC003ABCBD /* parse_execution.cpp in Sources */,
 				D0D02A86159839D5008E62BD /* postfork.cpp in Sources */,
 				D0D02A87159839D5008E62BD /* screen.cpp in Sources */,
 				D0D02A88159839D5008E62BD /* signal.cpp in Sources */,
@@ -1102,6 +1255,8 @@
 				D0D02A7A15983916008E62BD /* env_universal.cpp in Sources */,
 				D0D02A7B15983928008E62BD /* env_universal_common.cpp in Sources */,
 				D0D02A89159839DF008E62BD /* fish.cpp in Sources */,
+				D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */,
+				D0FE8EE8179FB760008C9F21 /* parse_productions.cpp in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -1329,6 +1484,74 @@
 			};
 			name = Release;
 		};
+		D08A328F17B4455100F3A533 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				ARCHS = "$(ARCHS_STANDARD_64_BIT)";
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				COPY_PHASE_STRIP = NO;
+				GCC_C_LANGUAGE_STANDARD = gnu99;
+				GCC_DYNAMIC_NO_PIC = NO;
+				GCC_ENABLE_OBJC_EXCEPTIONS = YES;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"DEBUG=1",
+					"$(inherited)",
+				);
+				GCC_WARN_UNINITIALIZED_AUTOS = YES;
+				MACOSX_DEPLOYMENT_TARGET = 10.8;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+			};
+			name = Debug;
+		};
+		D08A329017B4455100F3A533 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				ARCHS = "$(ARCHS_STANDARD_64_BIT)";
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				COPY_PHASE_STRIP = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu99;
+				GCC_ENABLE_OBJC_EXCEPTIONS = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES;
+				MACOSX_DEPLOYMENT_TARGET = 10.8;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+			};
+			name = Release;
+		};
+		D08A329117B4455100F3A533 /* Release_C++11 */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				ARCHS = "$(ARCHS_STANDARD_64_BIT)";
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				COPY_PHASE_STRIP = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu99;
+				GCC_ENABLE_OBJC_EXCEPTIONS = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES;
+				MACOSX_DEPLOYMENT_TARGET = 10.8;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+			};
+			name = "Release_C++11";
+		};
 		D0A084F813B3AC130099B651 /* Debug */ = {
 			isa = XCBuildConfiguration;
 			buildSettings = {
@@ -1605,6 +1828,16 @@
 			defaultConfigurationIsVisible = 0;
 			defaultConfigurationName = Release;
 		};
+		D08A329217B4455100F3A533 /* Build configuration list for PBXNativeTarget "fish_tests" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				D08A328F17B4455100F3A533 /* Debug */,
+				D08A329017B4455100F3A533 /* Release */,
+				D08A329117B4455100F3A533 /* Release_C++11 */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
 		D0A084F513B3AC130099B651 /* Build configuration list for PBXProject "fish" */ = {
 			isa = XCConfigurationList;
 			buildConfigurations = (

function.cpp

@@ -186,10 +186,15 @@ void function_add(const function_data_t &data, const parser_t &parser)
     /* Remove the old function */
     function_remove(data.name);
 
-
     /* Create and store a new function */
     const wchar_t *filename = reader_current_filename();
-    int def_offset = parser.line_number_of_character_at_offset(parser.current_block()->tok_pos) - 1;
+
+    int def_offset = -1;
+    if (parser.current_block() != NULL)
+    {
+        def_offset = parser.line_number_of_character_at_offset(parser.current_block()->tok_pos);
+    }
+
     const function_map_t::value_type new_pair(data.name, function_info_t(data, filename, def_offset, is_autoload));
     loaded_functions.insert(new_pair);
 

function.h

@@ -39,7 +39,7 @@ struct function_data_t
     /**
        Function definition
      */
-    wchar_t *definition;
+    const wchar_t *definition;
     /**
        List of all event handlers for this function
      */

highlight.h

@@ -9,66 +9,47 @@
 
 #include "env.h"
 #include "util.h"
-#include "screen.h"
 #include "color.h"
 
-/**
-   Internal value representing highlighting of normal text
-*/
-#define HIGHLIGHT_NORMAL 0x1
-/**
-   Internal value representing highlighting of an error
-*/
-#define HIGHLIGHT_ERROR 0x2
-/**
-   Internal value representing highlighting of a command
-*/
-#define HIGHLIGHT_COMMAND 0x4
-/**
-   Internal value representing highlighting of a process separator
-*/
-#define HIGHLIGHT_END 0x8
-/**
-   Internal value representing highlighting of a regular command parameter
-*/
-#define HIGHLIGHT_PARAM 0x10
-/**
-   Internal value representing highlighting of a comment
-*/
-#define HIGHLIGHT_COMMENT 0x20
-/**
-   Internal value representing highlighting of a matching parenteses, etc.
-*/
-#define HIGHLIGHT_MATCH 0x40
-/**
-   Internal value representing highlighting of a search match
-*/
-#define HIGHLIGHT_SEARCH_MATCH 0x80
-/**
-   Internal value representing highlighting of an operator
-*/
-#define HIGHLIGHT_OPERATOR 0x100
-/**
-   Internal value representing highlighting of an escape sequence
-*/
-#define HIGHLIGHT_ESCAPE 0x200
-/**
-   Internal value representing highlighting of a quoted string
-*/
-#define HIGHLIGHT_QUOTE 0x400
-/**
-   Internal value representing highlighting of an IO redirection
-*/
-#define HIGHLIGHT_REDIRECTION 0x800
-/**
-   Internal value representing highlighting a potentially valid path
-*/
-#define HIGHLIGHT_VALID_PATH 0x1000
+/* Internally, we specify highlight colors using a set of bits. Each highlight_spec is a 32 bit uint. We divide this into low 16 (foreground) and high 16 (background). Each half we further subdivide into low 8 (primary) and high 8 (modifiers). The primary is not a bitmask; specify exactly one. The modifiers are a bitmask; specify any number */
+enum
+{
+    /* The following values are mutually exclusive; specify at most one */
+    highlight_spec_normal = 0, // normal text
+    highlight_spec_error, // error
+    highlight_spec_command, //command
+    highlight_spec_statement_terminator, //process separator
+    highlight_spec_param, //command parameter (argument)
+    highlight_spec_comment, //comment
+    highlight_spec_match, //matching parenthesis, etc.
+    highlight_spec_search_match, //search match
+    highlight_spec_operator, //operator
+    highlight_spec_escape, //escape sequences
+    highlight_spec_quote, //quoted string
+    highlight_spec_redirection, //redirection
+    highlight_spec_autosuggestion, //autosuggestion
+
+    HIGHLIGHT_SPEC_PRIMARY_MASK = 0xFF,
+
+    /* The following values are modifiers */
+    highlight_modifier_valid_path = 0x100,
+
+    /* Very special value */
+    highlight_spec_invalid = 0xFFFF
+
+};
+typedef uint32_t highlight_spec_t;
+
+inline highlight_spec_t highlight_get_primary(highlight_spec_t val)
+{
+    return val & HIGHLIGHT_SPEC_PRIMARY_MASK;
+}
+
+inline highlight_spec_t highlight_make_background(highlight_spec_t val)
+{
+    return val << 16;
+}
 
-/**
-   Internal value representing highlighting an autosuggestion
-*/
-#define HIGHLIGHT_AUTOSUGGESTION 0x2000
 
 /**
    Internal value representing highlighting an active selection
@@ -88,7 +69,8 @@ struct file_detection_context_t;
    \param pos the cursor position. Used for quote matching, etc.
    \param error a list in which a description of each error will be inserted. May be 0, in whcich case no error descriptions will be generated.
 */
-void highlight_shell(const wcstring &buffstr, std::vector<int> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
+void highlight_shell(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
+void highlight_shell_new_parser(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
 
 /**
    Perform syntax highlighting for the text in buff. Matching quotes and paranthesis are highlighted. The result is
@@ -100,7 +82,7 @@ void highlight_shell(const wcstring &buffstr, std::vector<int> &color, size_t po
    \param pos the cursor position. Used for quote matching, etc.
    \param error a list in which a description of each error will be inserted. May be 0, in whcich case no error descriptions will be generated.
 */
-void highlight_universal(const wcstring &buffstr, std::vector<int> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
+void highlight_universal(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
 
 /**
    Translate from HIGHLIGHT_* to FISH_COLOR_* according to environment
@@ -109,10 +91,10 @@ void highlight_universal(const wcstring &buffstr, std::vector<int> &color, size_
    Example:
 
    If the environment variable FISH_FISH_COLOR_ERROR is set to 'red', a
-   call to highlight_get_color( HIGHLIGHT_ERROR) will return
+   call to highlight_get_color( highlight_error) will return
    FISH_COLOR_RED.
 */
-rgb_color_t highlight_get_color(int highlight, bool is_background);
+rgb_color_t highlight_get_color(highlight_spec_t highlight, bool is_background);
 
 /** Given a command 'str' from the history, try to determine whether we ought to suggest it by specially recognizing the command.
     Returns true if we validated the command. If so, returns by reference whether the suggestion is valid or not.
@@ -138,5 +120,9 @@ enum
 typedef unsigned int path_flags_t;
 bool is_potential_path(const wcstring &const_path, const wcstring_list_t &directories, path_flags_t flags, wcstring *out_path = NULL);
 
+/* For testing */
+void highlight_shell_classic(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
+void highlight_shell_new_parser(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
+
 #endif
 

history.h

@@ -61,6 +61,7 @@ public:
     {
         return contents;
     }
+
     bool empty() const
     {
         return contents.empty();

parse_constants.h

@@ -0,0 +1,264 @@
+/**\file parse_constants.h
+
+    Constants used in the programmatic representation of fish code.
+*/
+
+#ifndef fish_parse_constants_h
+#define fish_parse_constants_h
+
+#define PARSE_ASSERT(a) assert(a)
+#define PARSER_DIE() do { fprintf(stderr, "Parser dying!\n"); exit_without_destructors(-1); } while (0)
+
+
+enum parse_token_type_t
+{
+    token_type_invalid,
+
+    // Non-terminal tokens
+    symbol_job_list,
+    symbol_job,
+    symbol_job_continuation,
+    symbol_statement,
+    symbol_block_statement,
+    symbol_block_header,
+    symbol_for_header,
+    symbol_while_header,
+    symbol_begin_header,
+    symbol_function_header,
+
+    symbol_if_statement,
+    symbol_if_clause,
+    symbol_else_clause,
+    symbol_else_continuation,
+
+    symbol_switch_statement,
+    symbol_case_item_list,
+    symbol_case_item,
+
+    symbol_boolean_statement,
+    symbol_decorated_statement,
+    symbol_plain_statement,
+    symbol_arguments_or_redirections_list,
+    symbol_argument_or_redirection,
+
+    symbol_argument_list,
+
+    symbol_argument,
+    symbol_redirection,
+
+    symbol_optional_background,
+
+    symbol_end_command,
+
+    // Terminal types
+    parse_token_type_string,
+    parse_token_type_pipe,
+    parse_token_type_redirection,
+    parse_token_type_background,
+    parse_token_type_end,
+
+    // Special terminal type that means no more tokens forthcoming
+    parse_token_type_terminate,
+
+    // Very special terminal types that don't appear in the production list
+    parse_special_type_parse_error,
+    parse_special_type_tokenizer_error,
+    parse_special_type_comment,
+
+    FIRST_TERMINAL_TYPE = parse_token_type_string,
+    LAST_TERMINAL_TYPE = parse_token_type_terminate,
+
+    LAST_TOKEN_OR_SYMBOL = parse_token_type_terminate,
+    FIRST_PARSE_TOKEN_TYPE = parse_token_type_string
+};
+
+enum parse_keyword_t
+{
+    parse_keyword_none,
+    parse_keyword_if,
+    parse_keyword_else,
+    parse_keyword_for,
+    parse_keyword_in,
+    parse_keyword_while,
+    parse_keyword_begin,
+    parse_keyword_function,
+    parse_keyword_switch,
+    parse_keyword_case,
+    parse_keyword_end,
+    parse_keyword_and,
+    parse_keyword_or,
+    parse_keyword_not,
+    parse_keyword_command,
+    parse_keyword_builtin,
+
+    LAST_KEYWORD = parse_keyword_builtin
+};
+
+/* Statement decorations. This matches the order of productions in decorated_statement */
+enum parse_statement_decoration_t
+{
+    parse_statement_decoration_none,
+    parse_statement_decoration_command,
+    parse_statement_decoration_builtin
+};
+
+/* Parse error code list */
+enum parse_error_code_t
+{
+    parse_error_none,
+
+    /* Matching values from enum parser_error */
+    parse_error_syntax,
+    parse_error_eval,
+    parse_error_cmdsubst,
+
+    parse_error_generic, // unclassified error types
+
+    //tokenizer errors
+    parse_error_tokenizer_unterminated_quote,
+    parse_error_tokenizer_unterminated_subshell,
+    parse_error_tokenizer_unterminated_escape,
+    parse_error_tokenizer_other,
+
+    parse_error_unbalancing_end, //end outside of block
+    parse_error_unbalancing_else, //else outside of if
+    parse_error_unbalancing_case, //case outside of switch
+
+    parse_error_double_pipe, // foo || bar, has special error message
+    parse_error_double_background // foo && bar, has special error message
+};
+
+enum
+{
+    PARSER_TEST_ERROR = 1,
+    PARSER_TEST_INCOMPLETE = 2
+};
+typedef unsigned int parser_test_error_bits_t;
+
+
+/** Maximum number of function calls. */
+#define FISH_MAX_STACK_DEPTH 128
+
+/** Error message on a function that calls itself immediately */
+#define INFINITE_FUNC_RECURSION_ERR_MSG _( L"The function '%ls' calls itself immediately, which would result in an infinite loop.")
+
+
+/** Error message on reaching maximum call stack depth */
+#define CALL_STACK_LIMIT_EXCEEDED_ERR_MSG _( L"The function call stack limit has been exceeded. Do you have an accidental infinite loop?")
+
+/** Error message when a non-string token is found when expecting a command name */
+#define CMD_OR_ERR_MSG _( L"Expected a command, but instead found a pipe. Did you mean 'COMMAND; or COMMAND'? See the help section for the 'or' builtin command by typing 'help or'.")
+
+/** Error message when a non-string token is found when expecting a command name */
+#define CMD_AND_ERR_MSG _( L"Expected a command, but instead found a '&'. Did you mean 'COMMAND; and COMMAND'? See the help section for the 'and' builtin command by typing 'help and'.")
+
+/** Error message when encountering an illegal command name */
+#define ILLEGAL_CMD_ERR_MSG _( L"Illegal command name '%ls'")
+
+/** Error message when encountering an illegal file descriptor */
+#define ILLEGAL_FD_ERR_MSG _( L"Illegal file descriptor in redirection '%ls'")
+
+/** Error message for wildcards with no matches */
+#define WILDCARD_ERR_MSG _( L"No matches for wildcard '%ls'.")
+
+/** Error when using break outside of loop */
+#define INVALID_BREAK_ERR_MSG _( L"break command while not inside of loop" )
+
+/** Error when using continue outside of loop */
+#define INVALID_CONTINUE_ERR_MSG _( L"continue command while not inside of loop" )
+
+/** Error when using return builtin outside of function definition */
+#define INVALID_RETURN_ERR_MSG _( L"'return' builtin command outside of function definition" )
+
+/** Error message for Posix-style assignment: foo=bar */
+#define COMMAND_ASSIGN_ERR_MSG _( L"Unknown command '%ls'. Did you mean 'set %ls %ls'? See the help section on the set command by typing 'help set'.")
+
+/**
+   While block description
+*/
+#define WHILE_BLOCK N_( L"'while' block" )
+
+/**
+   For block description
+*/
+#define FOR_BLOCK N_( L"'for' block" )
+
+/**
+   Breakpoint block
+*/
+#define BREAKPOINT_BLOCK N_( L"Block created by breakpoint" )
+
+
+
+/**
+   If block description
+*/
+#define IF_BLOCK N_( L"'if' conditional block" )
+
+
+/**
+   Function definition block description
+*/
+#define FUNCTION_DEF_BLOCK N_( L"function definition block" )
+
+
+/**
+   Function invocation block description
+*/
+#define FUNCTION_CALL_BLOCK N_( L"function invocation block" )
+
+/**
+   Function invocation block description
+*/
+#define FUNCTION_CALL_NO_SHADOW_BLOCK N_( L"function invocation block with no variable shadowing" )
+
+
+/**
+   Switch block description
+*/
+#define SWITCH_BLOCK N_( L"'switch' block" )
+
+
+/**
+   Fake block description
+*/
+#define FAKE_BLOCK N_( L"unexecutable block" )
+
+
+/**
+   Top block description
+*/
+#define TOP_BLOCK N_( L"global root block" )
+
+
+/**
+   Command substitution block description
+*/
+#define SUBST_BLOCK N_( L"command substitution block" )
+
+
+/**
+   Begin block description
+*/
+#define BEGIN_BLOCK N_( L"'begin' unconditional block" )
+
+
+/**
+   Source block description
+*/
+#define SOURCE_BLOCK N_( L"Block created by the . builtin" )
+
+/**
+   Source block description
+*/
+#define EVENT_BLOCK N_( L"event handler block" )
+
+
+/**
+   Unknown block description
+*/
+#define UNKNOWN_BLOCK N_( L"unknown/invalid block" )
+
+
+
+#endif

parse_execution.h

@@ -0,0 +1,114 @@
+/**\file parse_execution.h
+
+   Provides the "linkage" between a parse_node_tree_t and actual execution structures (job_t, etc.).
+*/
+
+#ifndef FISH_PARSE_EXECUTION_H
+#define FISH_PARSE_EXECUTION_H
+
+#include "config.h"
+#include "util.h"
+#include "parse_tree.h"
+#include "proc.h"
+
+class job_t;
+struct profile_item_t;
+struct block_t;
+
+enum parse_execution_result_t
+{
+    /* The job was successfully executed (though it have failed on its own). */
+    parse_execution_success,
+
+    /* The job did not execute due to some error (e.g. failed to wildcard expand). An error will have been printed and proc_last_status will have been set. */
+    parse_execution_errored,
+
+    /* The job was cancelled (e.g. Ctrl-C) */
+    parse_execution_cancelled,
+
+    /* The job was skipped (e.g. due to a not-taken 'and' command). This is a special return allowed only from the populate functions, not the run functions. */
+    parse_execution_skipped
+};
+
+class parse_execution_context_t
+{
+private:
+    const parse_node_tree_t tree;
+    const wcstring src;
+    io_chain_t block_io;
+    parser_t * const parser;
+    //parse_error_list_t errors;
+
+    int eval_level;
+    std::vector<profile_item_t*> profile_items;
+
+    /* No copying allowed */
+    parse_execution_context_t(const parse_execution_context_t&);
+    parse_execution_context_t& operator=(const parse_execution_context_t&);
+
+    /* Should I cancel? */
+    bool should_cancel_execution(const block_t *block) const;
+
+    /* Ways that we can stop executing a block. These are in a sort of ascending order of importance, e.g. `exit` should trump `break` */
+    enum execution_cancellation_reason_t
+    {
+        execution_cancellation_none,
+        execution_cancellation_loop_control,
+        execution_cancellation_skip,
+        execution_cancellation_exit
+    };
+    execution_cancellation_reason_t cancellation_reason(const block_t *block) const;
+
+    /* Report an error. Always returns true. */
+    parse_execution_result_t report_error(const parse_node_t &node, const wchar_t *fmt, ...);
+    /* Wildcard error helper */
+    parse_execution_result_t report_unmatched_wildcard_error(const parse_node_t &unmatched_wildcard);
+
+    /* Command not found support */
+    void handle_command_not_found(const wcstring &cmd, const parse_node_t &statement_node, int err_code);
+
+    /* Utilities */
+    wcstring get_source(const parse_node_t &node) const;
+    const parse_node_t *get_child(const parse_node_t &parent, node_offset_t which, parse_token_type_t expected_type = token_type_invalid) const;
+    node_offset_t get_offset(const parse_node_t &node) const;
+    const parse_node_t *infinite_recursive_statement_in_job_list(const parse_node_t &job_list, wcstring *out_func_name) const;
+
+    /* Indicates whether a job is a simple block (one block, no redirections) */
+    bool job_is_simple_block(const parse_node_t &node) const;
+
+    enum process_type_t process_type_for_command(const parse_node_t &plain_statement, const wcstring &cmd) const;
+
+    /* These create process_t structures from statements */
+    parse_execution_result_t populate_job_process(job_t *job, process_t *proc, const parse_node_t &statement_node);
+    parse_execution_result_t populate_boolean_process(job_t *job, process_t *proc, const parse_node_t &bool_statement);
+    parse_execution_result_t populate_plain_process(job_t *job, process_t *proc, const parse_node_t &statement);
+    parse_execution_result_t populate_block_process(job_t *job, process_t *proc, const parse_node_t &statement_node);
+
+    /* These encapsulate the actual logic of various (block) statements. */
+    parse_execution_result_t run_block_statement(const parse_node_t &statement);
+    parse_execution_result_t run_for_statement(const parse_node_t &header, const parse_node_t &contents);
+    parse_execution_result_t run_if_statement(const parse_node_t &statement);
+    parse_execution_result_t run_switch_statement(const parse_node_t &statement);
+    parse_execution_result_t run_while_statement(const parse_node_t &header, const parse_node_t &contents);
+    parse_execution_result_t run_function_statement(const parse_node_t &header, const parse_node_t &contents);
+    parse_execution_result_t run_begin_statement(const parse_node_t &header, const parse_node_t &contents);
+
+    wcstring_list_t determine_arguments(const parse_node_t &parent, const parse_node_t **out_unmatched_wildcard_node);
+
+    /* Determines the IO chain. Returns true on success, false on error */
+    bool determine_io_chain(const parse_node_t &statement, io_chain_t *out_chain);
+
+    parse_execution_result_t run_1_job(const parse_node_t &job_node, const block_t *associated_block);
+    parse_execution_result_t run_job_list(const parse_node_t &job_list_node, const block_t *associated_block);
+    parse_execution_result_t populate_job_from_job_node(job_t *j, const parse_node_t &job_node, const block_t *associated_block);
+
+public:
+    parse_execution_context_t(const parse_node_tree_t &t, const wcstring &s, parser_t *p);
+
+    /* Start executing at the given node offset. Returns 0 if there was no error, 1 if there was an error */
+    parse_execution_result_t eval_node_at_offset(node_offset_t offset, const block_t *associated_block, const io_chain_t &io);
+
+};
+
+
+#endif

parse_productions.cpp

@@ -0,0 +1,550 @@
+#include "parse_productions.h"
+
+using namespace parse_productions;
+#define NO_PRODUCTION ((production_option_idx_t)(-1))
+
+static bool production_is_empty(const production_t production)
+{
+    return production[0] == token_type_invalid;
+}
+
+/* Empty productions are allowed but must be first. Validate that the given production is in the valid range, i.e. it is either not empty or there is a non-empty production after it */
+static bool production_is_valid(const production_options_t production_list, production_option_idx_t which)
+{
+    if (which < 0 || which >= MAX_PRODUCTIONS)
+        return false;
+
+    bool nonempty_found = false;
+    for (int i=which; i < MAX_PRODUCTIONS; i++)
+    {
+        if (! production_is_empty(production_list[i]))
+        {
+            nonempty_found = true;
+            break;
+        }
+    }
+    return nonempty_found;
+}
+
+#define PRODUCTIONS(sym) static const production_options_t productions_##sym
+#define RESOLVE(sym) static production_option_idx_t resolve_##sym (const parse_token_t &token1, const parse_token_t &token2)
+#define RESOLVE_ONLY(sym) static production_option_idx_t resolve_##sym (const parse_token_t &input1, const parse_token_t &input2) { return 0; }
+
+#define KEYWORD(x) ((x) + LAST_TOKEN_OR_SYMBOL + 1)
+
+
+/* A job_list is a list of jobs, separated by semicolons or newlines */
+PRODUCTIONS(job_list) =
+{
+    {},
+    {symbol_job, symbol_job_list},
+    {parse_token_type_end, symbol_job_list}
+};
+
+RESOLVE(job_list)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_string:
+            // some keywords are special
+            switch (token1.keyword)
+            {
+                case parse_keyword_end:
+                case parse_keyword_else:
+                case parse_keyword_case:
+                    // End this job list
+                    return 0;
+
+                default:
+                    // Normal string
+                    return 1;
+            }
+
+        case parse_token_type_pipe:
+        case parse_token_type_redirection:
+        case parse_token_type_background:
+            return 1;
+
+        case parse_token_type_end:
+            // Empty line
+            return 2;
+
+        case parse_token_type_terminate:
+            // no more commands, just transition to empty
+            return 0;
+
+        default:
+            return NO_PRODUCTION;
+    }
+}
+
+/* A job is a non-empty list of statements, separated by pipes. (Non-empty is useful for cases like if statements, where we require a command). To represent "non-empty", we require a statement, followed by a possibly empty job_continuation */
+
+PRODUCTIONS(job) =
+{
+    {symbol_statement, symbol_job_continuation}
+};
+RESOLVE_ONLY(job)
+
+PRODUCTIONS(job_continuation) =
+{
+    {},
+    {parse_token_type_pipe, symbol_statement, symbol_job_continuation}
+};
+RESOLVE(job_continuation)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_pipe:
+            // Pipe, continuation
+            return 1;
+
+        default:
+            // Not a pipe, no job continuation
+            return 0;
+    }
+}
+
+/* A statement is a normal command, or an if / while / and etc */
+PRODUCTIONS(statement) =
+{
+    {symbol_boolean_statement},
+    {symbol_block_statement},
+    {symbol_if_statement},
+    {symbol_switch_statement},
+    {symbol_decorated_statement}
+};
+RESOLVE(statement)
+{
+    /* The only block-like builtin that takes any parameters is 'function' So go to decorated statements if the subsequent token looks like '--'.
+        The logic here is subtle:
+            If we are 'begin', then we expect to be invoked with no arguments.
+            If we are 'function', then we are a non-block if we are invoked with -h or --help
+            If we are anything else, we require an argument, so do the same thing if the subsequent token is a statement terminator.
+    */
+
+    if (token1.type == parse_token_type_string)
+    {
+        // If we are a function, then look for help arguments
+        // Othewrise, if the next token looks like an option (starts with a dash), then parse it as a decorated statement
+        if (token1.keyword == parse_keyword_function && token2.is_help_argument)
+        {
+            return 4;
+        }
+        else if (token1.keyword != parse_keyword_function && token2.has_dash_prefix)
+        {
+            return 4;
+        }
+
+        // Likewise if the next token doesn't look like an argument at all. This corresponds to e.g. a "naked if".
+        bool naked_invocation_invokes_help = (token1.keyword != parse_keyword_begin && token1.keyword != parse_keyword_end);
+        if (naked_invocation_invokes_help && (token2.type == parse_token_type_end || token2.type == parse_token_type_terminate))
+        {
+            return 4;
+        }
+
+    }
+
+    switch (token1.type)
+    {
+        case parse_token_type_string:
+            switch (token1.keyword)
+            {
+                case parse_keyword_and:
+                case parse_keyword_or:
+                case parse_keyword_not:
+                    return 0;
+
+                case parse_keyword_for:
+                case parse_keyword_while:
+                case parse_keyword_function:
+                case parse_keyword_begin:
+                    return 1;
+
+                case parse_keyword_if:
+                    return 2;
+
+                case parse_keyword_else:
+                    return NO_PRODUCTION;
+
+                case parse_keyword_switch:
+                    return 3;
+
+                case parse_keyword_end:
+                    return NO_PRODUCTION;
+
+                // All other keywords fall through to decorated statement
+                default:
+                    return 4;
+            }
+            break;
+
+        case parse_token_type_pipe:
+        case parse_token_type_redirection:
+        case parse_token_type_background:
+        case parse_token_type_terminate:
+            return NO_PRODUCTION;
+            //parse_error(L"statement", token);
+
+        default:
+            return NO_PRODUCTION;
+    }
+}
+
+PRODUCTIONS(if_statement) =
+{
+    {symbol_if_clause, symbol_else_clause, symbol_end_command, symbol_arguments_or_redirections_list}
+};
+RESOLVE_ONLY(if_statement)
+
+PRODUCTIONS(if_clause) =
+{
+    { KEYWORD(parse_keyword_if), symbol_job, parse_token_type_end, symbol_job_list }
+};
+RESOLVE_ONLY(if_clause)
+
+PRODUCTIONS(else_clause) =
+{
+    { },
+    { KEYWORD(parse_keyword_else), symbol_else_continuation }
+};
+RESOLVE(else_clause)
+{
+    switch (token1.keyword)
+    {
+        case parse_keyword_else:
+            return 1;
+        default:
+            return 0;
+    }
+}
+
+PRODUCTIONS(else_continuation) =
+{
+    {symbol_if_clause, symbol_else_clause},
+    {parse_token_type_end, symbol_job_list}
+};
+RESOLVE(else_continuation)
+{
+    switch (token1.keyword)
+    {
+        case parse_keyword_if:
+            return 0;
+        default:
+            return 1;
+    }
+}
+
+PRODUCTIONS(switch_statement) =
+{
+    { KEYWORD(parse_keyword_switch), parse_token_type_string, parse_token_type_end, symbol_case_item_list, symbol_end_command, symbol_arguments_or_redirections_list}
+};
+RESOLVE_ONLY(switch_statement)
+
+PRODUCTIONS(case_item_list) =
+{
+    {},
+    {symbol_case_item, symbol_case_item_list},
+    {parse_token_type_end, symbol_case_item_list}
+};
+RESOLVE(case_item_list)
+{
+    if (token1.keyword == parse_keyword_case) return 1;
+    else if (token1.type == parse_token_type_end) return 2; //empty line
+    else return 0;
+}
+
+PRODUCTIONS(case_item) =
+{
+    {KEYWORD(parse_keyword_case), symbol_argument_list, parse_token_type_end, symbol_job_list}
+};
+RESOLVE_ONLY(case_item)
+
+PRODUCTIONS(argument_list) =
+{
+    {},
+    {symbol_argument, symbol_argument_list}
+};
+RESOLVE(argument_list)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_string:
+            return 1;
+        default:
+            return 0;
+    }
+}
+
+PRODUCTIONS(block_statement) =
+{
+    {symbol_block_header, parse_token_type_end, symbol_job_list, symbol_end_command, symbol_arguments_or_redirections_list}
+};
+RESOLVE_ONLY(block_statement)
+
+PRODUCTIONS(block_header) =
+{
+    {symbol_for_header},
+    {symbol_while_header},
+    {symbol_function_header},
+    {symbol_begin_header}
+};
+RESOLVE(block_header)
+{
+    switch (token1.keyword)
+    {
+        case parse_keyword_for:
+            return 0;
+        case parse_keyword_while:
+            return 1;
+        case parse_keyword_function:
+            return 2;
+        case parse_keyword_begin:
+            return 3;
+        default:
+            return NO_PRODUCTION;
+    }
+}
+
+PRODUCTIONS(for_header) =
+{
+    {KEYWORD(parse_keyword_for), parse_token_type_string, KEYWORD(parse_keyword_in), symbol_argument_list}
+};
+RESOLVE_ONLY(for_header)
+
+PRODUCTIONS(while_header) =
+{
+    {KEYWORD(parse_keyword_while), symbol_job}
+};
+RESOLVE_ONLY(while_header)
+
+PRODUCTIONS(begin_header) =
+{
+    {KEYWORD(parse_keyword_begin)}
+};
+RESOLVE_ONLY(begin_header)
+
+PRODUCTIONS(function_header) =
+{
+    {KEYWORD(parse_keyword_function), symbol_argument, symbol_argument_list}
+};
+RESOLVE_ONLY(function_header)
+
+/* A boolean statement is AND or OR or NOT */
+PRODUCTIONS(boolean_statement) =
+{
+    {KEYWORD(parse_keyword_and), symbol_statement},
+    {KEYWORD(parse_keyword_or), symbol_statement},
+    {KEYWORD(parse_keyword_not), symbol_statement}
+};
+RESOLVE(boolean_statement)
+{
+    switch (token1.keyword)
+    {
+        case parse_keyword_and:
+            return 0;
+        case parse_keyword_or:
+            return 1;
+        case parse_keyword_not:
+            return 2;
+        default:
+            return NO_PRODUCTION;
+    }
+}
+
+PRODUCTIONS(decorated_statement) =
+{
+    {symbol_plain_statement},
+    {KEYWORD(parse_keyword_command), symbol_plain_statement},
+    {KEYWORD(parse_keyword_builtin), symbol_plain_statement},
+};
+RESOLVE(decorated_statement)
+{
+    /* If this is e.g. 'command --help' then the command is 'command' and not a decoration. If the second token is not a string, then this is a naked 'command' and we should execute it as undecorated. */
+    if (token2.type != parse_token_type_string || token2.has_dash_prefix)
+    {
+        return 0;
+    }
+
+    switch (token1.keyword)
+    {
+        default:
+            return 0;
+        case parse_keyword_command:
+            return 1;
+        case parse_keyword_builtin:
+            return 2;
+    }
+}
+
+PRODUCTIONS(plain_statement) =
+{
+    {parse_token_type_string, symbol_arguments_or_redirections_list, symbol_optional_background}
+};
+RESOLVE_ONLY(plain_statement)
+
+PRODUCTIONS(arguments_or_redirections_list) =
+{
+    {},
+    {symbol_argument_or_redirection, symbol_arguments_or_redirections_list}
+};
+RESOLVE(arguments_or_redirections_list)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_string:
+        case parse_token_type_redirection:
+            return 1;
+        default:
+            return 0;
+    }
+}
+
+PRODUCTIONS(argument_or_redirection) =
+{
+    {symbol_argument},
+    {symbol_redirection}
+};
+RESOLVE(argument_or_redirection)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_string:
+            return 0;
+        case parse_token_type_redirection:
+            return 1;
+        default:
+            return NO_PRODUCTION;
+    }
+}
+
+PRODUCTIONS(argument) =
+{
+    {parse_token_type_string}
+};
+RESOLVE_ONLY(argument)
+
+PRODUCTIONS(redirection) =
+{
+    {parse_token_type_redirection, parse_token_type_string}
+};
+RESOLVE_ONLY(redirection)
+
+PRODUCTIONS(optional_background) =
+{
+    {},
+    { parse_token_type_background }
+};
+
+RESOLVE(optional_background)
+{
+    switch (token1.type)
+    {
+        case parse_token_type_background:
+            return 1;
+        default:
+            return 0;
+    }
+}
+
+PRODUCTIONS(end_command) =
+{
+    {KEYWORD(parse_keyword_end)}
+};
+RESOLVE_ONLY(end_command)
+
+#define TEST(sym) case (symbol_##sym): production_list = & productions_ ## sym ; resolver = resolve_ ## sym ; break;
+const production_t *parse_productions::production_for_token(parse_token_type_t node_type, const parse_token_t &input1, const parse_token_t &input2, production_option_idx_t *out_which_production, wcstring *out_error_text)
+{
+    bool log_it = false;
+    if (log_it)
+    {
+        fprintf(stderr, "Resolving production for %ls with input token <%ls>\n", token_type_description(node_type).c_str(), input1.describe().c_str());
+    }
+
+    /* Fetch the list of productions and the function to resolve them */
+    const production_options_t *production_list = NULL;
+    production_option_idx_t (*resolver)(const parse_token_t &input1, const parse_token_t &input2) = NULL;
+    switch (node_type)
+    {
+            TEST(job_list)
+            TEST(job)
+            TEST(statement)
+            TEST(job_continuation)
+            TEST(boolean_statement)
+            TEST(block_statement)
+            TEST(if_statement)
+            TEST(if_clause)
+            TEST(else_clause)
+            TEST(else_continuation)
+            TEST(switch_statement)
+            TEST(decorated_statement)
+            TEST(case_item_list)
+            TEST(case_item)
+            TEST(argument_list)
+            TEST(block_header)
+            TEST(for_header)
+            TEST(while_header)
+            TEST(begin_header)
+            TEST(function_header)
+            TEST(plain_statement)
+            TEST(arguments_or_redirections_list)
+            TEST(argument_or_redirection)
+            TEST(argument)
+            TEST(redirection)
+            TEST(optional_background)
+            TEST(end_command)
+
+        case parse_token_type_string:
+        case parse_token_type_pipe:
+        case parse_token_type_redirection:
+        case parse_token_type_background:
+        case parse_token_type_end:
+        case parse_token_type_terminate:
+            fprintf(stderr, "Terminal token type %ls passed to %s\n", token_type_description(node_type).c_str(), __FUNCTION__);
+            PARSER_DIE();
+            break;
+
+        case parse_special_type_parse_error:
+        case parse_special_type_tokenizer_error:
+        case parse_special_type_comment:
+            fprintf(stderr, "Special type %ls passed to %s\n", token_type_description(node_type).c_str(), __FUNCTION__);
+            PARSER_DIE();
+            break;
+
+
+        case token_type_invalid:
+            fprintf(stderr, "token_type_invalid passed to %s\n", __FUNCTION__);
+            PARSER_DIE();
+            break;
+
+    }
+    PARSE_ASSERT(production_list != NULL);
+    PARSE_ASSERT(resolver != NULL);
+
+    const production_t *result = NULL;
+    production_option_idx_t which = resolver(input1, input2);
+
+    if (log_it)
+    {
+        fprintf(stderr, "\tresolved to %u\n", (unsigned)which);
+    }
+
+
+    if (which == NO_PRODUCTION)
+    {
+        if (log_it)
+        {
+            fprintf(stderr, "Node type '%ls' has no production for input '%ls' (in %s)\n", token_type_description(node_type).c_str(), input1.describe().c_str(), __FUNCTION__);
+        }
+        result = NULL;
+    }
+    else
+    {
+        PARSE_ASSERT(production_is_valid(*production_list, which));
+        result = &((*production_list)[which]);
+    }
+    *out_which_production = which;
+    return result;
+}
+

parse_productions.h

@@ -0,0 +1,71 @@
+/**\file parse_tree.h
+
+    Programmatic representation of fish code.
+*/
+
+#ifndef FISH_PARSE_TREE_CONSTRUCTION_H
+#define FISH_PARSE_TREE_CONSTRUCTION_H
+
+#include "parse_tree.h"
+#include <inttypes.h>
+
+namespace parse_productions
+{
+
+#define MAX_PRODUCTIONS 5
+#define MAX_SYMBOLS_PER_PRODUCTION 6
+
+typedef uint32_t production_tag_t;
+
+/* A production is an array of unsigned char. Symbols are encoded directly as their symbol value. Keywords are encoded with an offset of LAST_TOKEN_OR_SYMBOL + 1. So essentially we glom together keywords and symbols. */
+typedef uint8_t production_element_t;
+
+/* An index into a production option list */
+typedef uint8_t production_option_idx_t;
+
+/* A production is an array of production elements */
+typedef production_element_t const production_t[MAX_SYMBOLS_PER_PRODUCTION];
+
+/* A production options is an array of (possible) productions */
+typedef production_t production_options_t[MAX_PRODUCTIONS];
+
+/* Resolve the type from a production element */
+inline parse_token_type_t production_element_type(production_element_t elem)
+{
+    if (elem > LAST_TOKEN_OR_SYMBOL)
+    {
+        return parse_token_type_string;
+    }
+    else
+    {
+        return static_cast<parse_token_type_t>(elem);
+    }
+}
+
+/* Resolve the keyword from a production element */
+inline parse_keyword_t production_element_keyword(production_element_t elem)
+{
+    if (elem > LAST_TOKEN_OR_SYMBOL)
+    {
+        // First keyword is LAST_TOKEN_OR_SYMBOL + 1
+        return static_cast<parse_keyword_t>(elem - LAST_TOKEN_OR_SYMBOL - 1);
+    }
+    else
+    {
+        return parse_keyword_none;
+    }
+}
+
+/* Check if an element is valid */
+inline bool production_element_is_valid(production_element_t elem)
+{
+    return elem != token_type_invalid;
+}
+
+/* Fetch a production. We are passed two input tokens. The first input token is guaranteed to not be invalid; the second token may be invalid if there's no more tokens. */
+const production_t *production_for_token(parse_token_type_t node_type, const parse_token_t &input1, const parse_token_t &input2, production_option_idx_t *out_which_production, wcstring *out_error_text);
+
+}
+
+
+#endif

parse_tree.h

@@ -0,0 +1,272 @@
+/**\file parse_tree.h
+
+    Programmatic representation of fish code.
+*/
+
+#ifndef FISH_PARSE_PRODUCTIONS_H
+#define FISH_PARSE_PRODUCTIONS_H
+
+#include <wchar.h>
+
+#include "config.h"
+#include "util.h"
+#include "common.h"
+#include "tokenizer.h"
+#include "parse_constants.h"
+#include <vector>
+#include <inttypes.h>
+
+class parse_node_t;
+class parse_node_tree_t;
+typedef size_t node_offset_t;
+#define NODE_OFFSET_INVALID (static_cast<node_offset_t>(-1))
+
+struct parse_error_t
+{
+    /** Text of the error */
+    wcstring text;
+
+    /** Code for the error */
+    enum parse_error_code_t code;
+
+    /** Offset and length of the token in the source code that triggered this error */
+    size_t source_start;
+    size_t source_length;
+
+    /** Return a string describing the error, suitable for presentation to the user. If skip_caret is false, the offending line with a caret is printed as well */
+    wcstring describe(const wcstring &src, bool skip_caret = false) const;
+};
+typedef std::vector<parse_error_t> parse_error_list_t;
+
+/* Returns a description of a list of parse errors */
+wcstring parse_errors_description(const parse_error_list_t &errors, const wcstring &src, const wchar_t *prefix = NULL);
+
+/** A struct representing the token type that we use internally */
+struct parse_token_t
+{
+    enum parse_token_type_t type; // The type of the token as represented by the parser
+    enum parse_keyword_t keyword; // Any keyword represented by this token
+    bool has_dash_prefix; // Hackish: whether the source contains a dash prefix
+    bool is_help_argument; // Hackish: whether the source looks like '-h' or '--help'
+    size_t source_start;
+    size_t source_length;
+
+    wcstring describe() const;
+    wcstring user_presentable_description() const;
+};
+
+
+enum
+{
+    parse_flag_none = 0,
+
+    /* Attempt to build a "parse tree" no matter what. This may result in a 'forest' of disconnected trees. This is intended to be used by syntax highlighting. */
+    parse_flag_continue_after_error = 1 << 0,
+
+    /* Include comment tokens */
+    parse_flag_include_comments = 1 << 1,
+
+    /* Indicate that the tokenizer should accept incomplete tokens */
+    parse_flag_accept_incomplete_tokens = 1 << 2,
+
+    /* Indicate that the parser should not generate the terminate token, allowing an 'unfinished' tree where some nodes may have no productions. */
+    parse_flag_leave_unterminated = 1 << 3
+
+};
+typedef unsigned int parse_tree_flags_t;
+
+wcstring parse_dump_tree(const parse_node_tree_t &tree, const wcstring &src);
+
+wcstring token_type_description(parse_token_type_t type);
+wcstring keyword_description(parse_keyword_t type);
+
+/** Class for nodes of a parse tree */
+class parse_node_t
+{
+public:
+
+    /* Type of the node */
+    enum parse_token_type_t type;
+
+    /* Start in the source code */
+    size_t source_start;
+
+    /* Length of our range in the source code */
+    size_t source_length;
+
+    /* Parent */
+    node_offset_t parent;
+
+    /* Children */
+    node_offset_t child_start;
+    uint8_t child_count;
+
+    /* Which production was used */
+    uint8_t production_idx;
+
+    /* Description */
+    wcstring describe(void) const;
+
+    /* Constructor */
+    explicit parse_node_t(parse_token_type_t ty) : type(ty), source_start(-1), source_length(0), parent(NODE_OFFSET_INVALID), child_start(0), child_count(0), production_idx(-1)
+    {
+    }
+
+    node_offset_t child_offset(node_offset_t which) const
+    {
+        PARSE_ASSERT(which < child_count);
+        return child_start + which;
+    }
+
+    /* Indicate if this node has a range of source code associated with it */
+    bool has_source() const
+    {
+        return source_start != (size_t)(-1);
+    }
+
+    /* Gets source for the node, or the empty string if it has no source */
+    wcstring get_source(const wcstring &str) const
+    {
+        if (! has_source())
+            return wcstring();
+        else
+            return wcstring(str, this->source_start, this->source_length);
+    }
+
+    /* Returns whether the given location is within the source range or at its end */
+    bool location_in_or_at_end_of_source_range(size_t loc) const
+    {
+        return has_source() && source_start <= loc && loc - source_start <= source_length;
+    }
+};
+
+
+/* The parse tree itself */
+class parse_node_tree_t : public std::vector<parse_node_t>
+{
+public:
+
+    /* Get the node corresponding to a child of the given node, or NULL if there is no such child. If expected_type is provided, assert that the node has that type.
+     */
+    const parse_node_t *get_child(const parse_node_t &parent, node_offset_t which, parse_token_type_t expected_type = token_type_invalid) const;
+
+    /* Find the first direct child of the given node of the given type. asserts on failure
+     */
+    const parse_node_t &find_child(const parse_node_t &parent, parse_token_type_t type) const;
+
+    /* Get the node corresponding to the parent of the given node, or NULL if there is no such child. If expected_type is provided, only returns the parent if it is of that type. Note the asymmetry: get_child asserts since the children are known, but get_parent does not, since the parent may not be known. */
+    const parse_node_t *get_parent(const parse_node_t &node, parse_token_type_t expected_type = token_type_invalid) const;
+
+    /* Returns the first ancestor of the given type, or NULL. */
+    const parse_node_t *get_first_ancestor_of_type(const parse_node_t &node, parse_token_type_t desired_type) const;
+
+    /* Find all the nodes of a given type underneath a given node, up to max_count of them */
+    typedef std::vector<const parse_node_t *> parse_node_list_t;
+    parse_node_list_t find_nodes(const parse_node_t &parent, parse_token_type_t type, size_t max_count = (size_t)(-1)) const;
+
+    /* Finds the last node of a given type underneath a given node, or NULL if it could not be found. If parent is NULL, this finds the last node in the tree of that type. */
+    const parse_node_t *find_last_node_of_type(parse_token_type_t type, const parse_node_t *parent = NULL) const;
+
+    /* Finds a node containing the given source location. If 'parent' is not NULL, it must be an ancestor. */
+    const parse_node_t *find_node_matching_source_location(parse_token_type_t type, size_t source_loc, const parse_node_t *parent) const;
+
+    /* Indicate if the given argument_list or arguments_or_redirections_list is a root list, or has a parent */
+    bool argument_list_is_root(const parse_node_t &node) const;
+
+    /* Utilities */
+
+    /* Given a plain statement, get the decoration (from the parent node), or none if there is no decoration */
+    enum parse_statement_decoration_t decoration_for_plain_statement(const parse_node_t &node) const;
+
+    /* Given a plain statement, get the command by reference (from the child node). Returns true if successful. Clears the command on failure. */
+    bool command_for_plain_statement(const parse_node_t &node, const wcstring &src, wcstring *out_cmd) const;
+
+    /* Given a plain statement, return true if the statement is part of a pipeline. If include_first is set, the first command in a pipeline is considered part of it; otherwise only the second or additional commands are */
+    bool statement_is_in_pipeline(const parse_node_t &node, bool include_first) const;
+
+    /* Given a redirection, get the redirection type (or TOK_NONE) and target (file path, or fd) */
+    enum token_type type_for_redirection(const parse_node_t &node, const wcstring &src, int *out_fd, wcstring *out_target) const;
+
+    /* If the given node is a block statement, returns the header node (for_header, while_header, begin_header, or function_header). Otherwise returns NULL */
+    const parse_node_t *header_node_for_block_statement(const parse_node_t &node) const;
+
+    /* Given a node list (e.g. of type symbol_job_list) and a node type (e.g. symbol_job), return the next element of the given type in that list, and the tail (by reference). Returns NULL if we've exhausted the list. */
+    const parse_node_t *next_node_in_node_list(const parse_node_t &node_list, parse_token_type_t item_type, const parse_node_t **list_tail) const;
+
+    /* Given a job, return all of its statements. These are 'specific statements' (e.g. symbol_decorated_statement, not symbol_statement) */
+    parse_node_list_t specific_statements_for_job(const parse_node_t &job) const;
+};
+
+/* The big entry point. Parse a string! */
+bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags, parse_node_tree_t *output, parse_error_list_t *errors);
+
+/* Fish grammar:
+
+# A job_list is a list of jobs, separated by semicolons or newlines
+
+    job_list = <empty> |
+                job job_list
+                <TOK_END> job_list
+
+# A job is a non-empty list of statements, separated by pipes. (Non-empty is useful for cases like if statements, where we require a command). To represent "non-empty", we require a statement, followed by a possibly empty job_continuation
+
+    job = statement job_continuation
+    job_continuation = <empty> |
+                       <TOK_PIPE> statement job_continuation
+
+# A statement is a normal command, or an if / while / and etc
+
+    statement = boolean_statement | block_statement | if_statement | switch_statement | decorated_statement
+
+# A block is a conditional, loop, or begin/end
+
+    if_statement = if_clause else_clause end_command arguments_or_redirections_list
+    if_clause = <IF> job STATEMENT_TERMINATOR job_list
+    else_clause = <empty> |
+                 <ELSE> else_continuation
+    else_continuation = if_clause else_clause |
+                        STATEMENT_TERMINATOR job_list
+
+    switch_statement = SWITCH <TOK_STRING> STATEMENT_TERMINATOR case_item_list end_command arguments_or_redirections_list
+    case_item_list = <empty> |
+                    case_item case_item_list |
+                    <TOK_END> case_item_list
+
+    case_item = CASE argument_list STATEMENT_TERMINATOR job_list
+
+    block_statement = block_header <TOK_END> job_list end_command arguments_or_redirections_list
+    block_header = for_header | while_header | function_header | begin_header
+    for_header = FOR var_name IN argument_list
+    while_header = WHILE job
+    begin_header = BEGIN
+
+# Functions take arguments, and require at least one (the name). No redirections allowed.
+    function_header = FUNCTION argument argument_list
+
+# A boolean statement is AND or OR or NOT
+
+    boolean_statement = AND statement | OR statement | NOT statement
+
+# A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command"
+
+    decorated_statement = plain_statement | COMMAND plain_statement | BUILTIN plain_statement
+    plain_statement = <TOK_STRING> arguments_or_redirections_list optional_background
+
+    argument_list = <empty> | argument argument_list
+
+    arguments_or_redirections_list = <empty> |
+                                     argument_or_redirection arguments_or_redirections_list
+    argument_or_redirection = argument | redirection
+    argument = <TOK_STRING>
+
+    redirection = <TOK_REDIRECTION> <TOK_STRING>
+
+    terminator = <TOK_END> | <TOK_BACKGROUND>
+
+    optional_background = <empty> | <TOK_BACKGROUND>
+
+    end_command = END
+
+*/
+
+#endif

parse_util.cpp

@@ -38,18 +38,13 @@
 #include "env.h"
 #include "signal.h"
 #include "wildcard.h"
+#include "parse_tree.h"
+#include "parser.h"
 
 /**
-   Maximum number of autoloaded items opf a specific type to keep in
-   memory at a time.
+   Error message for improper use of the exec builtin
 */
-#define AUTOLOAD_MAX 10
-
-/**
-   Minimum time, in seconds, before an autoloaded item will be
-   unloaded
-*/
-#define AUTOLOAD_MIN_AGE 60
+#define EXEC_ERR_MSG _(L"The '%ls' command can not be used in a pipeline")
 
 int parse_util_lineno(const wchar_t *str, size_t offset)
 {
@@ -164,7 +159,7 @@ int parse_util_locate_cmdsubst(const wchar_t *in, wchar_t **begin, wchar_t **end
 
     CHECK(in, 0);
 
-    for (pos = (wchar_t *)in; *pos; pos++)
+    for (pos = const_cast<wchar_t *>(in); *pos; pos++)
     {
         if (prev != '\\')
         {
@@ -240,6 +235,42 @@ int parse_util_locate_cmdsubst(const wchar_t *in, wchar_t **begin, wchar_t **end
     return 1;
 }
 
+int parse_util_locate_cmdsubst_range(const wcstring &str, size_t *inout_cursor_offset, wcstring *out_contents, size_t *out_start, size_t *out_end, bool accept_incomplete)
+{
+    /* Clear the return values */
+    out_contents->clear();
+    *out_start = 0;
+    *out_end = str.size();
+
+    /* Nothing to do if the offset is at or past the end of the string. */
+    if (*inout_cursor_offset >= str.size())
+        return 0;
+
+    /* Defer to the wonky version */
+    const wchar_t * const buff = str.c_str();
+    const wchar_t * const valid_range_start = buff + *inout_cursor_offset, *valid_range_end = buff + str.size();
+    wchar_t *cmdsub_begin = NULL, *cmdsub_end = NULL;
+    int ret = parse_util_locate_cmdsubst(valid_range_start, &cmdsub_begin, &cmdsub_end, accept_incomplete);
+    if (ret > 0)
+    {
+        /* The command substitutions must not be NULL and must be in the valid pointer range, and the end must be bigger than the beginning */
+        assert(cmdsub_begin != NULL && cmdsub_begin >= valid_range_start && cmdsub_begin <= valid_range_end);
+        assert(cmdsub_end != NULL && cmdsub_end > cmdsub_begin && cmdsub_end >= valid_range_start && cmdsub_end <= valid_range_end);
+
+        /* Assign the substring to the out_contents */
+        const wchar_t *interior_begin = cmdsub_begin + 1;
+        out_contents->assign(interior_begin, cmdsub_end - interior_begin);
+
+        /* Return the start and end */
+        *out_start = cmdsub_begin - buff;
+        *out_end = cmdsub_end - buff;
+
+        /* Update the inout_cursor_offset. Note this may cause it to exceed str.size(), though overflow is not likely */
+        *inout_cursor_offset = 1 + *out_end;
+    }
+    return ret;
+}
+
 void parse_util_cmdsubst_extent(const wchar_t *buff, size_t cursor_pos, const wchar_t **a, const wchar_t **b)
 {
     const wchar_t * const cursor = buff + cursor_pos;
@@ -768,3 +799,353 @@ wcstring parse_util_escape_string_with_quote(const wcstring &cmd, wchar_t quote)
     }
     return result;
 }
+
+/* We are given a parse tree, the index of a node within the tree, its indent, and a vector of indents the same size as the original source string. Set the indent correspdonding to the node's source range, if appropriate.
+
+   trailing_indent is the indent for nodes with unrealized source, i.e. if I type 'if false <ret>' then we have an if node with an empty job list (without source) but we want the last line to be indented anyways.
+
+   switch statements also indent.
+
+   max_visited_node_idx is the largest index we visited.
+*/
+static void compute_indents_recursive(const parse_node_tree_t &tree, node_offset_t node_idx, int node_indent, parse_token_type_t parent_type, std::vector<int> *indents, int *trailing_indent, node_offset_t *max_visited_node_idx)
+{
+    /* Guard against incomplete trees */
+    if (node_idx > tree.size())
+        return;
+
+    /* Update max_visited_node_idx */
+    if (node_idx > *max_visited_node_idx)
+        *max_visited_node_idx = node_idx;
+
+    /* We could implement this by utilizing the fish grammar. But there's an easy trick instead: almost everything that wraps a job list should be indented by 1. So just find all of the job lists. One exception is switch; the other exception is job_list itself: a job_list is a job and a job_list, and we want that child list to be indented the same as the parent. So just find all job_lists whose parent is not a job_list, and increment their indent by 1. */
+
+    const parse_node_t &node = tree.at(node_idx);
+    const parse_token_type_t node_type = node.type;
+
+    /* Increment the indent if we are either a root job_list, or root case_item_list */
+    const bool is_root_job_list = (node_type == symbol_job_list && parent_type != symbol_job_list);
+    const bool is_root_case_item_list = (node_type == symbol_case_item_list && parent_type != symbol_case_item_list);
+    if (is_root_job_list || is_root_case_item_list)
+    {
+        node_indent += 1;
+    }
+
+    /* If we have source, store the trailing indent unconditionally. If we do not have source, store the trailing indent only if ours is bigger; this prevents the trailing "run" of terminal job lists from affecting the trailing indent. For example, code like this:
+
+            if foo
+
+      will be parsed as this:
+
+      job_list
+        job
+           if_statement
+               job [if]
+               job_list [empty]
+         job_list [empty]
+
+      There's two "terminal" job lists, and we want the innermost one.
+
+      Note we are relying on the fact that nodes are in the same order as the source, i.e. an in-order traversal of the node tree also traverses the source from beginning to end.
+    */
+    if (node.has_source() || node_indent > *trailing_indent)
+    {
+        *trailing_indent = node_indent;
+    }
+
+
+    /* Store the indent into the indent array */
+    if (node.has_source())
+    {
+        assert(node.source_start < indents->size());
+        indents->at(node.source_start) = node_indent;
+    }
+
+
+    /* Recursive to all our children */
+    for (node_offset_t idx = 0; idx < node.child_count; idx++)
+    {
+        /* Note we pass our type to our child, which becomes its parent node type */
+        compute_indents_recursive(tree, node.child_start + idx, node_indent, node_type, indents, trailing_indent, max_visited_node_idx);
+    }
+}
+
+std::vector<int> parse_util_compute_indents(const wcstring &src)
+{
+    /* Make a vector the same size as the input string, which contains the indents. Initialize them to -1. */
+    const size_t src_size = src.size();
+    std::vector<int> indents(src_size, -1);
+
+    /* Parse the string. We pass continue_after_error to produce a forest; the trailing indent of the last node we visited becomes the input indent of the next. I.e. in the case of 'switch foo ; cas', we get an invalid parse tree (since 'cas' is not valid) but we indent it as if it were a case item list */
+    parse_node_tree_t tree;
+    parse_tree_from_string(src, parse_flag_continue_after_error | parse_flag_accept_incomplete_tokens, &tree, NULL /* errors */);
+
+    /* Start indenting at the first node. If we have a parse error, we'll have to start indenting from the top again */
+    node_offset_t start_node_idx = 0;
+    int last_trailing_indent = 0;
+
+    while (start_node_idx < tree.size())
+    {
+        /* The indent that we'll get for the last line */
+        int trailing_indent = 0;
+
+        /* Biggest offset we visited */
+        node_offset_t max_visited_node_idx = 0;
+
+        /* Invoke the recursive version. As a hack, pass job_list for the 'parent' token type, which will prevent the really-root job list from indenting */
+        compute_indents_recursive(tree, start_node_idx, last_trailing_indent, symbol_job_list, &indents, &trailing_indent, &max_visited_node_idx);
+
+        /* We may have more to indent. The trailing indent becomes our current indent. Start at the node after the last we visited. */
+        last_trailing_indent = trailing_indent;
+        start_node_idx = max_visited_node_idx + 1;
+    }
+
+    int last_indent = 0;
+    for (size_t i=0; i<src_size; i++)
+    {
+        int this_indent = indents.at(i);
+        if (this_indent < 0)
+        {
+            indents.at(i) = last_indent;
+        }
+        else
+        {
+            /* New indent level */
+            last_indent = this_indent;
+            /* Make all whitespace before a token have the new level. This avoid using the wrong indentation level if a new line starts with whitespace. */
+            size_t prev_char_idx = i;
+            while (prev_char_idx--)
+            {
+                if (!wcschr(L" \n\t\r", src.at(prev_char_idx)))
+                    break;
+                indents.at(prev_char_idx) = last_indent;
+            }
+        }
+    }
+
+    /* Ensure trailing whitespace has the trailing indent. This makes sure a new line is correctly indented even if it is empty. */
+    size_t suffix_idx = src_size;
+    while (suffix_idx--)
+    {
+        if (!wcschr(L" \n\t\r", src.at(suffix_idx)))
+            break;
+        indents.at(suffix_idx) = last_trailing_indent;
+    }
+
+    return indents;
+}
+
+/* Append a syntax error to the given error list */
+static bool append_syntax_error(parse_error_list_t *errors, const parse_node_t &node, const wchar_t *fmt, ...)
+{
+    parse_error_t error;
+    error.source_start = node.source_start;
+    error.source_length = node.source_length;
+    error.code = parse_error_syntax;
+
+    va_list va;
+    va_start(va, fmt);
+    error.text = vformat_string(fmt, va);
+    va_end(va);
+
+    errors->push_back(error);
+    return true;
+}
+
+/**
+   Returns 1 if the specified command is a builtin that may not be used in a pipeline
+*/
+static int parser_is_pipe_forbidden(const wcstring &word)
+{
+    return contains(word,
+                    L"exec",
+                    L"case",
+                    L"break",
+                    L"return",
+                    L"continue");
+}
+
+// Check if the first argument under the given node is --help
+static bool first_argument_is_help(const parse_node_tree_t &node_tree, const parse_node_t &node, const wcstring &src)
+{
+    bool is_help = false;
+    const parse_node_tree_t::parse_node_list_t arg_nodes = node_tree.find_nodes(node, symbol_argument, 1);
+    if (! arg_nodes.empty())
+    {
+        // Check the first argument only
+        const parse_node_t &arg = *arg_nodes.at(0);
+        const wcstring first_arg_src = arg.get_source(src);
+        is_help = parser_t::is_help(first_arg_src.c_str(), 3);
+    }
+    return is_help;
+}
+
+parser_test_error_bits_t parse_util_detect_errors(const wcstring &buff_src, parse_error_list_t *out_errors)
+{
+    parse_node_tree_t node_tree;
+    parse_error_list_t parse_errors;
+
+    // Whether we encountered a parse error
+    bool errored = false;
+
+    // Whether we encountered an unclosed block
+    // We detect this via an 'end_command' block without source
+    bool has_unclosed_block = false;
+
+    // Whether there's an unclosed quote, and therefore unfinished
+    bool has_unclosed_quote = false;
+
+    // Parse the input string into a parse tree
+    // Some errors are detected here
+    bool parsed = parse_tree_from_string(buff_src, parse_flag_leave_unterminated, &node_tree, &parse_errors);
+
+    for (size_t i=0; i < parse_errors.size(); i++)
+    {
+        if (parse_errors.at(i).code == parse_error_tokenizer_unterminated_quote)
+        {
+            // Remove this error, since we don't consider it a real error
+            has_unclosed_quote = true;
+            parse_errors.erase(parse_errors.begin() + i);
+            i--;
+        }
+    }
+    // #1238: If the only error was unterminated quote, then consider this to have parsed successfully. A better fix would be to have parse_tree_from_string return this information directly (but it would be a shame to munge up its nice bool return).
+    if (parse_errors.empty() && has_unclosed_quote)
+        parsed = true;
+
+    if (! parsed)
+    {
+        errored = true;
+    }
+
+    // Expand all commands
+    // Verify 'or' and 'and' not used inside pipelines
+    // Verify pipes via parser_is_pipe_forbidden
+    // Verify return only within a function
+
+    if (! errored)
+    {
+        const size_t node_tree_size = node_tree.size();
+        for (size_t i=0; i < node_tree_size; i++)
+        {
+            const parse_node_t &node = node_tree.at(i);
+            if (node.type == symbol_end_command && ! node.has_source())
+            {
+                // an 'end' without source is an unclosed block
+                has_unclosed_block = true;
+            }
+            else if (node.type == symbol_boolean_statement)
+            {
+                // 'or' and 'and' can be in a pipeline, as long as they're first
+                // These numbers 0 and 1 correspond to productions for boolean_statement. This should be cleaned up.
+                bool is_and = (node.production_idx == 0), is_or = (node.production_idx == 1);
+                if ((is_and || is_or) && node_tree.statement_is_in_pipeline(node, false /* don't count first */))
+                {
+                    errored = append_syntax_error(&parse_errors, node, EXEC_ERR_MSG, is_and ? L"and" : L"or");
+                }
+            }
+            else if (node.type == symbol_plain_statement)
+            {
+                wcstring command;
+                if (node_tree.command_for_plain_statement(node, buff_src, &command))
+                {
+                    // Check that we can expand the command
+                    if (! expand_one(command, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES | EXPAND_SKIP_JOBS))
+                    {
+                        errored = append_syntax_error(&parse_errors, node, ILLEGAL_CMD_ERR_MSG, command.c_str());
+                    }
+
+                    // Check that pipes are sound
+                    if (! errored && parser_is_pipe_forbidden(command))
+                    {
+                        // forbidden commands cannot be in a pipeline at all
+                        if (node_tree.statement_is_in_pipeline(node, true /* count first */))
+                        {
+                            errored = append_syntax_error(&parse_errors, node, EXEC_ERR_MSG, command.c_str());
+                        }
+                    }
+
+                    // Check that we don't return from outside a function
+                    // But we allow it if it's 'return --help'
+                    if (! errored && command == L"return")
+                    {
+                        const parse_node_t *ancestor = &node;
+                        bool found_function = false;
+                        while (ancestor != NULL)
+                        {
+                            const parse_node_t *possible_function_header = node_tree.header_node_for_block_statement(*ancestor);
+                            if (possible_function_header != NULL && possible_function_header->type == symbol_function_header)
+                            {
+                                found_function = true;
+                                break;
+                            }
+                            ancestor = node_tree.get_parent(*ancestor);
+
+                        }
+                        if (! found_function && ! first_argument_is_help(node_tree, node, buff_src))
+                        {
+                            errored = append_syntax_error(&parse_errors, node, INVALID_RETURN_ERR_MSG);
+                        }
+                    }
+
+                    // Check that we don't break or continue from outside a loop
+                    if (! errored && (command == L"break" || command == L"continue"))
+                    {
+                        // Walk up until we hit a 'for' or 'while' loop. If we hit a function first, stop the search; we can't break an outer loop from inside a function.
+                        // This is a little funny because we can't tell if it's a 'for' or 'while' loop from the ancestor alone; we need the header. That is, we hit a block_statement, and have to check its header.
+                        bool found_loop = false, end_search = false;
+                        const parse_node_t *ancestor = &node;
+                        while (ancestor != NULL && ! end_search)
+                        {
+                            const parse_node_t *loop_or_function_header = node_tree.header_node_for_block_statement(*ancestor);
+                            if (loop_or_function_header != NULL)
+                            {
+                                switch (loop_or_function_header->type)
+                                {
+                                    case symbol_while_header:
+                                    case symbol_for_header:
+                                        // this is a loop header, so we can break or continue
+                                        found_loop = true;
+                                        end_search = true;
+                                        break;
+
+                                    case symbol_function_header:
+                                        // this is a function header, so we cannot break or continue. We stop our search here.
+                                        found_loop = false;
+                                        end_search = true;
+                                        break;
+
+                                    default:
+                                        // most likely begin / end style block, which makes no difference
+                                        break;
+                                }
+                            }
+                            ancestor = node_tree.get_parent(*ancestor);
+                        }
+
+                        if (! found_loop && ! first_argument_is_help(node_tree, node, buff_src))
+                        {
+                            errored = append_syntax_error(&parse_errors, node, (command == L"break" ? INVALID_BREAK_ERR_MSG : INVALID_CONTINUE_ERR_MSG));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    parser_test_error_bits_t res = 0;
+
+    if (errored)
+        res |= PARSER_TEST_ERROR;
+
+    if (has_unclosed_block || has_unclosed_quote)
+        res |= PARSER_TEST_INCOMPLETE;
+
+    if (out_errors)
+    {
+        out_errors->swap(parse_errors);
+    }
+
+    return res;
+
+}

parse_util.h

@@ -8,6 +8,7 @@
 #define FISH_PARSE_UTIL_H
 
 #include "autoload.h"
+#include "parse_tree.h"
 #include <wchar.h>
 #include <map>
 #include <set>
@@ -27,6 +28,25 @@ int parse_util_locate_cmdsubst(const wchar_t *in,
                                wchar_t **end,
                                bool accept_incomplete);
 
+/**
+   Alternative API. Iterate over command substitutions.
+
+   \param str the string to search for subshells
+   \param inout_cursor_offset On input, the location to begin the search. On output, either the end of the string, or just after the closed-paren.
+   \param out_contents On output, the contents of the command substitution
+   \param out_start On output, the offset of the start of the command substitution (open paren)
+   \param out_end On output, the offset of the end of the command substitution (close paren), or the end of the string if it was incomplete
+   \param accept_incomplete whether to permit missing closing parenthesis
+   \return -1 on syntax error, 0 if no subshells exist and 1 on sucess
+*/
+
+int parse_util_locate_cmdsubst_range(const wcstring &str,
+                                     size_t *inout_cursor_offset,
+                                     wcstring *out_contents,
+                                     size_t *out_start,
+                                     size_t *out_end,
+                                     bool accept_incomplete);
+
 /**
    Find the beginning and end of the command substitution under the
    cursor. If no subshell is found, the entire string is returned. If
@@ -140,5 +160,9 @@ void parse_util_get_parameter_info(const wcstring &cmd, const size_t pos, wchar_
 */
 wcstring parse_util_escape_string_with_quote(const wcstring &cmd, wchar_t quote);
 
+/** Given a string, parse it as fish code and then return the indents. The return value has the same size as the string */
+std::vector<int> parse_util_compute_indents(const wcstring &src);
+
+parser_test_error_bits_t parse_util_detect_errors(const wcstring &buff_src, parse_error_list_t *out_errors = NULL);
 
 #endif

parser.h

@@ -11,11 +11,9 @@
 #include "util.h"
 #include "event.h"
 #include "function.h"
+#include "parse_tree.h"
 #include <vector>
 
-#define PARSER_TEST_ERROR 1
-#define PARSER_TEST_INCOMPLETE 2
-
 /**
    event_blockage_t represents a block on events of the specified type
 */
@@ -98,37 +96,18 @@ public:
     bool had_command; /**< Set to non-zero once a command has been executed in this block */
     int tok_pos; /**< The start index of the block */
 
-    /**
-       Status for the current loop block. Can be any of the values from the loop_status enum.
-    */
+    node_offset_t node_offset; /* Offset of the node */
+
+    /** Status for the current loop block. Can be any of the values from the loop_status enum. */
     int loop_status;
 
-    /**
-       The job that is currently evaluated in the specified block.
-    */
+    /** The job that is currently evaluated in the specified block. */
     job_t *job;
 
-#if 0
-    union
-    {
-        int while_state;  /**< True if the loop condition has not yet been evaluated*/
-        wchar_t *for_variable; /**< Name of the variable to loop over */
-        int if_state; /**< The state of the if block, can be one of IF_STATE_UNTESTED, IF_STATE_FALSE, IF_STATE_TRUE */
-        wchar_t *switch_value; /**< The value to test in a switch block */
-        const wchar_t *source_dest; /**< The name of the file to source*/
-        event_t *event; /**<The event that triggered this block */
-        wchar_t *function_call_name;
-    } param1;
-#endif
-
-    /**
-       Name of file that created this block
-    */
+    /** Name of file that created this block */
     const wchar_t *src_filename;
 
-    /**
-       Line number where this block was created
-    */
+    /** Line number where this block was created */
     int src_lineno;
 
     /** Whether we should pop the environment variable stack when we're popped off of the block stack */
@@ -291,9 +270,11 @@ struct profile_item_t
 };
 
 struct tokenizer_t;
+class parse_execution_context_t;
 
 class parser_t
 {
+    friend class parse_execution_context_t;
 private:
     enum parser_type_t parser_type;
 
@@ -306,6 +287,12 @@ private:
     /** Position of last error */
     int err_pos;
 
+    /** Indication that we should skip all blocks */
+    bool cancellation_requested;
+
+    /** Stack of execution contexts. We own these pointers and must delete them */
+    std::vector<parse_execution_context_t *> execution_contexts;
+
     /** Description of last error */
     wcstring err_buff;
 
@@ -326,7 +313,7 @@ private:
 
     /** The jobs associated with this parser */
     job_list_t my_job_list;
-    
+
     /** The list of blocks, allocated with new. It's our responsibility to delete these */
     std::vector<block_t *> block_stack;
 
@@ -341,6 +328,7 @@ private:
     parser_t(const parser_t&);
     parser_t& operator=(const parser_t&);
 
+
     void parse_job_argument_list(process_t *p, job_t *j, tokenizer_t *tok, std::vector<completion_t>&, bool);
     int parse_job(process_t *p, job_t *j, tokenizer_t *tok);
     void skipped_exec(job_t * j);
@@ -350,7 +338,10 @@ private:
     void print_errors_stderr();
 
     /** Create a job */
-    job_t *job_create();
+    job_t *job_create(const io_chain_t &io);
+
+    /** Adds a job to the beginning of the job list. */
+    void job_add(job_t *job);
 
 public:
     std::vector<profile_item_t*> profile_items;
@@ -389,11 +380,15 @@ public:
 
       \return 0 on success, 1 otherwise
     */
-    int eval(const wcstring &cmdStr, const io_chain_t &io, enum block_type_t block_type);
+    int eval(const wcstring &cmd_str, const io_chain_t &io, enum block_type_t block_type);
+    int eval_new_parser(const wcstring &cmd, const io_chain_t &io, enum block_type_t block_type);
+
+    /** Evaluates a block node at the given node offset in the topmost execution context */
+    int eval_block_node(node_offset_t node_idx, const io_chain_t &io, enum block_type_t block_type);
 
     /**
       Evaluate line as a list of parameters, i.e. tokenize it and perform parameter expansion and cmdsubst execution on the tokens.
-      The output is inserted into output, and should be freed by the caller.
+      The output is inserted into output.
 
       \param line Line to evaluate
       \param output List to insert output to
@@ -402,7 +397,7 @@ public:
       \param line Line to evaluate
       \param output List to insert output to
     */
-    int eval_args(const wchar_t *line, std::vector<completion_t> &output);
+    void eval_args(const wchar_t *line, std::vector<completion_t> &output);
 
     /**
        Sets the current evaluation error. This function should only be used by libraries that are called by
@@ -411,7 +406,7 @@ public:
        \param p The character offset at which the error occured
        \param str The printf-style error message filter
     */
-    void error(int ec, int p, const wchar_t *str, ...);
+    void error(int ec, size_t p, const wchar_t *str, ...);
 
     /**
        Returns a string describing the current parser pisition in the format 'FILENAME (line LINE_NUMBER): LINE'.
@@ -435,15 +430,15 @@ public:
 
     /** Set the current position in the latest string of the tokenizer. */
     void set_pos(int p);
-    
+
     /** Returns the block at the given index. 0 corresponds to the innermost block. Returns NULL when idx is at or equal to the number of blocks. */
     const block_t *block_at_index(size_t idx) const;
     block_t *block_at_index(size_t idx);
-    
+
     /** Returns the current (innermost) block */
     const block_t *current_block() const;
     block_t *current_block();
-    
+
     /** Count of blocks */
     size_t block_count() const
     {
@@ -465,6 +460,9 @@ public:
     /** Remove the outermost block namespace */
     void pop_block();
 
+    /** Remove the outermost block, asserting it's the given one */
+    void pop_block(const block_t *b);
+
     /** Return a description of the given blocktype */
     const wchar_t *get_block_desc(int block) const;
 
@@ -492,7 +490,7 @@ public:
        \param out if non-null, any errors in the command will be filled out into this buffer
        \param prefix the prefix string to prepend to each error message written to the \c out buffer
     */
-    int test(const wchar_t * buff, int *block_level = NULL, wcstring *out = NULL, const wchar_t *prefix = NULL);
+    void get_backtrace(const wcstring &src, const parse_error_list_t &errors, wcstring *output) const;
 
     /**
        Test if the specified string can be parsed as an argument list,
@@ -529,7 +527,7 @@ public:
        \param s the string to test
        \param min_match is the minimum number of characters that must match in a long style option, i.e. the longest common prefix between --help and any other option. If less than 3, 3 will be assumed.
     */
-    int is_help(const wchar_t *s, int min_match) const;
+    static int is_help(const wchar_t *s, int min_match);
 
     /**
        Returns the file currently evaluated by the parser. This can be
@@ -541,11 +539,14 @@ public:
     /**
        Write a stack trace starting at the specified block to the specified wcstring
     */
-    void stack_trace(size_t block_idx, wcstring &buff);
+    void stack_trace(size_t block_idx, wcstring &buff) const;
 
     int get_block_type(const wchar_t *cmd) const;
     const wchar_t *get_block_command(int type) const;
 };
 
+/* Temporary */
+bool parser_use_ast(void);
+
 
 #endif

proc.cpp

@@ -136,7 +136,9 @@ static bool proc_had_barrier = false;
 int get_is_interactive(void)
 {
     ASSERT_IS_MAIN_THREAD();
-    return is_interactive;
+    /* is_interactive is initialized to -1; ensure someone has popped/pushed it before then */
+    assert(is_interactive >= 0);
+    return is_interactive > 0;
 }
 
 bool get_proc_had_barrier()
@@ -515,7 +517,8 @@ static void handle_child_status(pid_t pid, int status)
 process_t::process_t() :
     argv_array(),
     argv0_narrow(),
-    type(0),
+    type(),
+    internal_block_node(NODE_OFFSET_INVALID),
     actual_cmd(),
     pid(0),
     pipe_write_fd(0),
@@ -638,6 +641,9 @@ int job_reap(bool interactive)
 
     locked++;
 
+    /* Preserve the exit status */
+    const int saved_status = proc_get_last_status();
+
     /*
       job_read may fire an event handler, we do not want to call
       ourselves recursively (to avoid infinite recursion).
@@ -752,6 +758,9 @@ int job_reap(bool interactive)
     if (found)
         fflush(stdout);
 
+    /* Restore the exit status. */
+    proc_set_last_status(saved_status);
+
     locked = 0;
 
     return found;

proc.h

@@ -20,6 +20,7 @@
 #include "util.h"
 #include "io.h"
 #include "common.h"
+#include "parse_tree.h"
 
 /**
    The status code use when a command was not found
@@ -54,7 +55,7 @@
 /**
    Types of processes
 */
-enum
+enum process_type_t
 {
     /**
        A regular external command
@@ -72,6 +73,10 @@ enum
        A block of commands
     */
     INTERNAL_BLOCK,
+
+    /** A block of commands, represented as a node */
+    INTERNAL_BLOCK_NODE,
+
     /**
        The exec builtin
     */
@@ -81,8 +86,7 @@ enum
     */
     INTERNAL_BUFFER,
 
-}
-;
+};
 
 enum
 {
@@ -151,8 +155,10 @@ public:
       INTERNAL_BUILTIN, \c INTERNAL_FUNCTION, \c INTERNAL_BLOCK,
       INTERNAL_EXEC, or INTERNAL_BUFFER
     */
-    int type;
+    enum process_type_t type;
 
+    /* For internal block processes only, the node offset of the block */
+    node_offset_t internal_block_node;
 
     /** Sets argv */
     void set_argv(const wcstring_list_t &argv)
@@ -505,18 +511,12 @@ void job_free(job_t* j);
 */
 void job_promote(job_t *job);
 
-/**
-   Create a new job.
-*/
-job_t *job_create();
-
 /**
   Return the job with the specified job id.
   If id is 0 or less, return the last job used.
 */
 job_t *job_get(job_id_t id);
 
-
 /**
   Return the job with the specified pid.
 */

reader.cpp

@@ -99,6 +99,7 @@ commence.
 #include "path.h"
 #include "parse_util.h"
 #include "parser_keywords.h"
+#include "parse_tree.h"
 
 /**
    Maximum length of prefix string when printing completion
@@ -186,9 +187,6 @@ static volatile unsigned int s_generation_count;
 /* This pthreads generation count is set when an autosuggestion background thread starts up, so it can easily check if the work it is doing is no longer useful. */
 static pthread_key_t generation_count_key;
 
-/* A color is an int */
-typedef int color_t;
-
 static void set_command_line_and_position(const wcstring &new_str, size_t pos);
 
 /**
@@ -292,7 +290,7 @@ public:
        color[i] is the classification (according to the enum in
        highlight.h) of buff[i].
     */
-    std::vector<color_t> colors;
+    std::vector<highlight_spec_t> colors;
 
     /** An array defining the block level at each character. */
     std::vector<int> indents;
@@ -563,7 +561,7 @@ wcstring combine_command_and_autosuggestion(const wcstring &cmdline, const wcstr
 static void reader_repaint()
 {
     // Update the indentation
-    parser_t::principal_parser().test(data->command_line.c_str(), &data->indents[0]);
+    data->indents = parse_util_compute_indents(data->command_line);
 
     // Combine the command and autosuggestion into one string
     wcstring full_line = combine_command_and_autosuggestion(data->command_line, data->autosuggestion);
@@ -572,8 +570,8 @@ static void reader_repaint()
     if (len < 1)
         len = 1;
 
-    std::vector<color_t> colors = data->colors;
-    colors.resize(len, HIGHLIGHT_AUTOSUGGESTION);
+    std::vector<highlight_spec_t> colors = data->colors;
+    colors.resize(len, highlight_spec_autosuggestion);
 
     if(data->sel_active)
     {
@@ -693,7 +691,7 @@ void reader_data_t::command_line_changed()
     size_t len = command_length();
 
     /* When we grow colors, propagate the last color (if any), under the assumption that usually it will be correct. If it is, it avoids a repaint. */
-    color_t last_color = colors.empty() ? color_t() : colors.back();
+    highlight_spec_t last_color = colors.empty() ? highlight_spec_t() : colors.back();
     colors.resize(len, last_color);
 
     indents.resize(len);
@@ -716,117 +714,55 @@ bool reader_expand_abbreviation_in_command(const wcstring &cmdline, size_t curso
     const size_t subcmd_offset = cmdsub_begin - buff;
 
     const wcstring subcmd = wcstring(cmdsub_begin, cmdsub_end - cmdsub_begin);
-    const wchar_t *subcmd_cstr = subcmd.c_str();
+    const size_t subcmd_cursor_pos = cursor_pos - subcmd_offset;
 
-    /* Get the token containing the cursor */
-    const wchar_t *subcmd_tok_begin = NULL, *subcmd_tok_end = NULL;
-    assert(cursor_pos >= subcmd_offset);
-    size_t subcmd_cursor_pos = cursor_pos - subcmd_offset;
-    parse_util_token_extent(subcmd_cstr, subcmd_cursor_pos, &subcmd_tok_begin, &subcmd_tok_end, NULL, NULL);
+    /* Parse this subcmd */
+    parse_node_tree_t parse_tree;
+    parse_tree_from_string(subcmd, parse_flag_continue_after_error | parse_flag_accept_incomplete_tokens, &parse_tree, NULL);
 
-    /* Compute the offset of the token before the cursor within the subcmd */
-    assert(subcmd_tok_begin >= subcmd_cstr);
-    assert(subcmd_tok_end >= subcmd_tok_begin);
-    const size_t subcmd_tok_begin_offset = subcmd_tok_begin - subcmd_cstr;
-    const size_t subcmd_tok_length = subcmd_tok_end - subcmd_tok_begin;
-
-    /* Now parse the subcmd, looking for commands */
-    bool had_cmd = false, previous_token_is_cmd = false;
-    tokenizer_t tok(subcmd_cstr, TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
-    for (; tok_has_next(&tok); tok_next(&tok))
+    /* Look for plain statements where the cursor is at the end of the command */
+    const parse_node_t *matching_cmd_node = NULL;
+    const size_t len = parse_tree.size();
+    for (size_t i=0; i < len; i++)
     {
-        size_t tok_pos = static_cast<size_t>(tok_get_pos(&tok));
-        if (tok_pos > subcmd_tok_begin_offset)
+        const parse_node_t &node = parse_tree.at(i);
+
+        /* Only interested in plain statements with source */
+        if (node.type != symbol_plain_statement || ! node.has_source())
+            continue;
+
+        /* Skip decorated statements */
+        if (parse_tree.decoration_for_plain_statement(node) != parse_statement_decoration_none)
+            continue;
+
+        /* Get the command node. Skip it if we can't or it has no source */
+        const parse_node_t *cmd_node = parse_tree.get_child(node, 0, parse_token_type_string);
+        if (cmd_node == NULL || ! cmd_node->has_source())
+            continue;
+
+        /* Now see if its source range contains our cursor, including at the end */
+        if (subcmd_cursor_pos >= cmd_node->source_start && subcmd_cursor_pos <= cmd_node->source_start + cmd_node->source_length)
         {
-            /* We've passed the token we're interested in */
+            /* Success! */
+            matching_cmd_node = cmd_node;
             break;
         }
-
-        int last_type = tok_last_type(&tok);
-
-        switch (last_type)
-        {
-            case TOK_STRING:
-            {
-                if (had_cmd)
-                {
-                    /* Parameter to the command. */
-                }
-                else
-                {
-                    const wcstring potential_cmd = tok_last(&tok);
-                    if (parser_keywords_is_subcommand(potential_cmd))
-                    {
-                        if (potential_cmd == L"command" || potential_cmd == L"builtin")
-                        {
-                            /* 'command' and 'builtin' defeat abbreviation expansion. Skip this command. */
-                            had_cmd = true;
-                        }
-                        else
-                        {
-                            /* Other subcommand. Pretend it doesn't exist so that we can expand the following command */
-                            had_cmd = false;
-                        }
-                    }
-                    else
-                    {
-                        /* It's a normal command */
-                        had_cmd = true;
-                        if (tok_pos == subcmd_tok_begin_offset)
-                        {
-                            /* This is the token we care about! */
-                            previous_token_is_cmd = true;
-                        }
-                    }
-                }
-                break;
-            }
-
-            case TOK_REDIRECT_NOCLOB:
-            case TOK_REDIRECT_OUT:
-            case TOK_REDIRECT_IN:
-            case TOK_REDIRECT_APPEND:
-            case TOK_REDIRECT_FD:
-            {
-                if (!had_cmd)
-                {
-                    break;
-                }
-                tok_next(&tok);
-                break;
-            }
-
-            case TOK_PIPE:
-            case TOK_BACKGROUND:
-            case TOK_END:
-            {
-                had_cmd = false;
-                break;
-            }
-
-            case TOK_COMMENT:
-            case TOK_ERROR:
-            default:
-            {
-                break;
-            }
-        }
     }
 
+    /* Now if we found a command node, expand it */
     bool result = false;
-    if (previous_token_is_cmd)
+    if (matching_cmd_node != NULL)
     {
-        /* The token is a command. Try expanding it as an abbreviation. */
-        const wcstring token = wcstring(subcmd, subcmd_tok_begin_offset, subcmd_tok_length);
+        assert(matching_cmd_node->type == parse_token_type_string);
+        const wcstring token = matching_cmd_node->get_source(subcmd);
         wcstring abbreviation;
         if (expand_abbreviation(token, &abbreviation))
         {
             /* There was an abbreviation! Replace the token in the full command. Maintain the relative position of the cursor. */
             if (output != NULL)
             {
-                size_t cmd_tok_begin_offset = subcmd_tok_begin_offset + subcmd_offset;
                 output->assign(cmdline);
-                output->replace(cmd_tok_begin_offset, subcmd_tok_length, abbreviation);
+                output->replace(subcmd_offset + matching_cmd_node->source_start, matching_cmd_node->source_length, abbreviation);
             }
             result = true;
         }
@@ -1551,7 +1487,7 @@ struct autosuggestion_context_t
         {
             const completion_t &comp = completions.at(0);
             size_t cursor = this->cursor_pos;
-            this->autosuggestion = completion_apply_to_command_line(comp.completion.c_str(), comp.flags, this->search_string, &cursor, true /* append only */);
+            this->autosuggestion = completion_apply_to_command_line(comp.completion, comp.flags, this->search_string, &cursor, true /* append only */);
             return 1;
         }
 
@@ -1641,7 +1577,7 @@ static void reader_flash()
 
     for (size_t i=0; i<data->buff_pos; i++)
     {
-        data->colors.at(i) = HIGHLIGHT_SEARCH_MATCH<<16;
+        data->colors.at(i) = highlight_spec_search_match<<16;
     }
 
     reader_repaint();
@@ -2186,11 +2122,9 @@ static void reader_interactive_destroy()
 
 void reader_sanity_check()
 {
-    if (get_is_interactive())
+    /* Note: 'data' is non-null if we are interactive, except in the testing environment */
+    if (get_is_interactive() && data != NULL)
     {
-        if (!data)
-            sanity_lose();
-
         if (!(data->buff_pos <= data->command_length()))
             sanity_lose();
 
@@ -2320,7 +2254,6 @@ static void handle_token_history(int forward, int reset)
             */
             if (data->history_search.go_backwards())
             {
-                wcstring item = data->history_search.current_string();
                 data->token_history_buff = data->history_search.current_string();
             }
             current_pos = data->token_history_buff.size();
@@ -2610,30 +2543,26 @@ void reader_run_command(parser_t &parser, const wcstring &cmd)
 
 int reader_shell_test(const wchar_t *b)
 {
-    int res = parser_t::principal_parser().test(b);
+    assert(b != NULL);
+    wcstring bstr = b;
+
+    /* Append a newline, to act as a statement terminator */
+    bstr.push_back(L'\n');
+
+    parse_error_list_t errors;
+    int res = parse_util_detect_errors(bstr, &errors);
 
     if (res & PARSER_TEST_ERROR)
     {
-        wcstring sb;
+        wcstring error_desc;
+        parser_t::principal_parser().get_backtrace(bstr, errors, &error_desc);
 
-        const int tmp[1] = {0};
-        const int tmp2[1] = {0};
-        const wcstring empty;
-
-        s_write(&data->screen,
-                empty,
-                empty,
-                empty,
-                0,
-                tmp,
-                tmp2,
-                0,
-                0,
-                0);
-
-
-        parser_t::principal_parser().test(b, NULL, &sb, L"fish");
-        fwprintf(stderr, L"%ls", sb.c_str());
+        // ensure we end with a newline. Also add an initial newline, because it's likely the user just hit enter and so there's junk on the current line
+        if (! string_suffixes_string(L"\n", error_desc))
+        {
+            error_desc.push_back(L'\n');
+        }
+        fwprintf(stderr, L"\n%ls", error_desc.c_str());
     }
     return res;
 }
@@ -2766,7 +2695,7 @@ public:
     const wcstring string_to_highlight;
 
     /** Color buffer */
-    std::vector<color_t> colors;
+    std::vector<highlight_spec_t> colors;
 
     /** The position to use for bracket matching */
     const size_t match_highlight_pos;
@@ -2821,7 +2750,7 @@ static void highlight_search(void)
             size_t end = match_pos + needle.size();
             for (size_t i=match_pos; i < end; i++)
             {
-                data->colors.at(i) |= (HIGHLIGHT_SEARCH_MATCH<<16);
+                data->colors.at(i) |= (highlight_spec_search_match<<16);
             }
         }
     }
@@ -2883,10 +2812,10 @@ static void reader_super_highlight_me_plenty(size_t match_highlight_pos)
 }
 
 
-int exit_status()
+bool shell_is_exiting()
 {
     if (get_is_interactive())
-        return job_list_is_empty() && data->end_loop;
+        return job_list_is_empty() && data != NULL && data->end_loop;
     else
         return end_loop;
 }
@@ -3137,6 +3066,7 @@ const wchar_t *reader_readline(void)
             is_interactive_read = 1;
             c=input_readch();
             is_interactive_read = was_interactive_read;
+            //fprintf(stderr, "C: %lx\n", (long)c);
 
             if (((!wchar_private(c))) && (c>31) && (c != 127))
             {
@@ -3312,6 +3242,9 @@ const wchar_t *reader_readline(void)
                     const wchar_t *token_begin, *token_end;
                     parse_util_token_extent(cmdsub_begin, data->buff_pos - (cmdsub_begin-buff), &token_begin, &token_end, 0, 0);
 
+                    /* Hack: the token may extend past the end of the command substitution, e.g. in (echo foo) the last token is 'foo)'. Don't let that happen. */
+                    if (token_end > cmdsub_end) token_end = cmdsub_end;
+
                     /* Figure out how many steps to get from the current position to the end of the current token. */
                     size_t end_of_token_offset = token_end - buff;
 
@@ -3466,7 +3399,7 @@ const wchar_t *reader_readline(void)
                     {
                         //history_reset();
                         data->history_search.go_to_end();
-                        reader_set_buffer(data->search_buff.c_str(), data->search_buff.size());
+                        reader_set_buffer(data->search_buff, data->search_buff.size());
                     }
                     else
                     {
@@ -3543,12 +3476,9 @@ const wchar_t *reader_readline(void)
                     case 0:
                     {
                         /* Finished command, execute it. Don't add items that start with a leading space. */
-                        if (! data->command_line.empty() && data->command_line.at(0) != L' ')
+                        if (data->history != NULL && ! data->command_line.empty() && data->command_line.at(0) != L' ')
                         {
-                            if (data->history != NULL)
-                            {
-                                data->history->add_with_file_detection(data->command_line);
-                            }
+                            data->history->add_with_file_detection(data->command_line);
                         }
                         finished=1;
                         update_buff_pos(data->command_length());
@@ -4059,13 +3989,15 @@ static int read_ni(int fd, const io_chain_t &io)
             res = 1;
         }
 
-        wcstring sb;
-        if (! parser.test(str.c_str(), 0, &sb, L"fish"))
+        parse_error_list_t errors;
+        if (! parse_util_detect_errors(str, &errors))
         {
             parser.eval(str, io, TOP);
         }
         else
         {
+            wcstring sb;
+            parser.get_backtrace(str, errors, &sb);
             fwprintf(stderr, L"%ls", sb.c_str());
             res = 1;
         }

reader.h

@@ -16,6 +16,7 @@
 #include "io.h"
 #include "common.h"
 #include "complete.h"
+#include "highlight.h"
 
 class parser_t;
 class completion_t;
@@ -180,7 +181,7 @@ void reader_set_complete_function(complete_function_t);
  The type of a highlight function.
  */
 class env_vars_snapshot_t;
-typedef void (*highlight_function_t)(const wcstring &, std::vector<int> &, size_t, wcstring_list_t *, const env_vars_snapshot_t &vars);
+typedef void (*highlight_function_t)(const wcstring &, std::vector<highlight_spec_t> &, size_t, wcstring_list_t *, const env_vars_snapshot_t &vars);
 
 /**
  Specify function for syntax highlighting. The function must take these arguments:
@@ -224,7 +225,7 @@ void reader_set_exit_on_interrupt(bool flag);
 /**
    Returns true if the shell is exiting, 0 otherwise.
 */
-int exit_status();
+bool shell_is_exiting();
 
 /**
    The readers interrupt signal handler. Cancels all currently running blocks.

screen.cpp

@@ -668,7 +668,7 @@ static void s_move(screen_t *s, data_buffer_t *b, int new_x, int new_y)
 /**
    Set the pen color for the terminal
 */
-static void s_set_color(screen_t *s, data_buffer_t *b, int c)
+static void s_set_color(screen_t *s, data_buffer_t *b, highlight_spec_t c)
 {
     scoped_buffer_t scoped_buffer(b);
 
@@ -1215,7 +1215,8 @@ static screen_layout_t compute_layout(screen_t *s,
         // If the command wraps, and the prompt is not short, place the command on its own line.
         // A short prompt is 33% or less of the terminal's width.
         const size_t prompt_percent_width = (100 * left_prompt_width) / screen_width;
-        if (left_prompt_width + first_command_line_width + 1 > screen_width && prompt_percent_width > 33) {
+        if (left_prompt_width + first_command_line_width + 1 > screen_width && prompt_percent_width > 33)
+        {
             result.prompts_get_own_line = true;
         }
 
@@ -1232,7 +1233,7 @@ void s_write(screen_t *s,
              const wcstring &right_prompt,
              const wcstring &commandline,
              size_t explicit_len,
-             const int *colors,
+             const highlight_spec_t *colors,
              const int *indent,
              size_t cursor_pos,
              size_t sel_start_pos,

screen.h

@@ -13,6 +13,7 @@
 #define FISH_SCREEN_H
 
 #include <vector>
+#include "highlight.h"
 
 /**
    A class representing a single line of a screen.
@@ -20,7 +21,7 @@
 struct line_t
 {
     std::vector<wchar_t> text;
-    std::vector<int> colors;
+    std::vector<highlight_spec_t> colors;
     bool is_soft_wrapped;
 
     line_t() : text(), colors(), is_soft_wrapped(false)
@@ -33,7 +34,7 @@ struct line_t
         colors.clear();
     }
 
-    void append(wchar_t txt, int color)
+    void append(wchar_t txt, highlight_spec_t color)
     {
         text.push_back(txt);
         colors.push_back(color);
@@ -49,7 +50,7 @@ struct line_t
         return text.at(idx);
     }
 
-    int color_at(size_t idx) const
+    highlight_spec_t color_at(size_t idx) const
     {
         return colors.at(idx);
     }
@@ -189,7 +190,7 @@ void s_write(screen_t *s,
              const wcstring &right_prompt,
              const wcstring &commandline,
              size_t explicit_len,
-             const int *colors,
+             const highlight_spec_t *colors,
              const int *indent,
              size_t cursor_pos,
              size_t sel_start_pos,

share/completions/lsusb.fish

@@ -0,0 +1,7 @@
+complete -c lsusb -s v -l verbose --description "Increase verbosity (show descriptors)"
+complete -x -c lsusb -s s -a '(__fish_complete_lsusb)' --description "Show only devices with specified device and/or bus numbers (in decimal)"
+complete -c lsusb -s d --description "Show only devices with the specified vendor and product ID numbers (in hexadecimal)"
+complete -c lsusb -s D -l device --description "Selects which device lsusb will examine"
+complete -c lsusb -s t -l tree --description "Dump the physical USB device hierarchy as a tree"
+complete -c lsusb -s V -l version --description "Show version of program"
+complete -c lsusb -s h -l help --description "Show usage and help"

share/functions/__fish_complete_lsusb.fish

@@ -0,0 +1,3 @@
+function __fish_complete_lsusb
+	lsusb | awk '{print $2 ":" $4}'| cut -c1-7
+end

share/functions/__fish_print_hostnames.fish

@@ -16,10 +16,15 @@ function __fish_print_hostnames -d "Print a list of known hostnames"
 	# Does not match hostnames with @directives specified
 	sgrep -Eoh '^[^#@|, ]*' ~/.ssh/known_hosts{,2} ^/dev/null
 
+	# Print hosts from system wide ssh configuration file
+	if [ -e /etc/ssh/ssh_config ]
+		# Ignore lines containing wildcards
+		sgrep -Eoi '^ *host[^*]*$' /etc/ssh/ssh_config | cut -d '=' -f 2 | tr ' ' '\n'
+	end
+
 	# Print hosts from ssh configuration file
 	if [ -e ~/.ssh/config ]
 		# Ignore lines containing wildcards
 		sgrep -Eoi '^ *host[^*]*$' ~/.ssh/config | cut -d '=' -f 2 | tr ' ' '\n'
 	end
 end
-

share/functions/__fish_print_packages.fish

@@ -67,7 +67,7 @@ function __fish_print_packages
 		end
 
 		# Remove package version information from output and pipe into cache file
-		rpm -qa >$cache_file |sed -e 's/-[^-]*-[^-]*$//' | sed -e 's/$/'\t$package'/' &
+		rpm -qa | sed -e 's/-[^-]*-[^-]*$//' | sed -e 's/$/'\t$package'/' >$cache_file &
 	end
 
 	# This completes the package name from the portage tree.

share/functions/fish_default_key_bindings.fish

@@ -78,6 +78,7 @@ function fish_default_key_bindings -d "Default (Emacs-like) key bindings for fis
 	bind $argv \ct transpose-chars
 	bind $argv \et transpose-words
 	bind $argv \eu upcase-word
+
 	# This clashes with __fish_list_current_token
 	# bind $argv \el downcase-word
 	bind $argv \ec capitalize-word
@@ -107,6 +108,9 @@ function fish_default_key_bindings -d "Default (Emacs-like) key bindings for fis
 	bind $argv \ed 'set -l cmd (commandline); if test -z "$cmd"; echo; dirh; commandline -f repaint; else; commandline -f kill-word; end'
 	bind $argv \cd delete-or-exit
 
+	bind \ed forward-kill-word
+	bind \ed kill-word
+
 	# Allow reading manpages by pressing F1
 	bind $argv -k f1 'man (basename (commandline -po; echo))[1] ^/dev/null; or echo -n \a'
 

share/tools/web_config/fishconfig.css

@@ -0,0 +1,461 @@
+body {
+    background-color: #292939;
+    font-family: Courier, "Courier New", monospace;
+    color: white;
+}
+
+#ancestor {
+    width: 800px;
+    margin-left: auto;
+    margin-right: auto;
+    margin-top: 25px;
+}
+
+#parent {
+    width: 100%;
+    min-height: 480px;
+    margin-top: 12px;
+}
+
+#tab_parent {
+    display: table;
+    width: 100%;
+    height: 50px;;
+}
+
+.tab {
+	display: table-cell;
+    border: 1px solid #111;
+    border-right: none;
+    padding-bottom: 15px;
+    padding-top: 15px;
+    font-size: 17pt;
+    text-align: center;
+    width: 15%;
+    background-color: #292929;
+    cursor: pointer;
+}
+
+#tab_parent :first-child {
+	border-top-left-radius: 8px;
+}
+
+#tab_parent :last-child {
+	border-right: 1px solid #111;
+	border-top-right-radius: 8px;
+}
+
+.tab_first {
+	border-top-left-radius: 5px;
+	border-bottom-left-radius: 5px;
+}
+
+.tab_last {
+	border-top-right-radius: 5px;
+	border-bottom-right-radius: 5px;
+}
+
+.selected_tab {
+    background-color: black;
+    border-color: black;
+}
+
+#tab_contents {
+	padding-top: 35px;
+	width: 100%;
+	background-color: black;
+	border-bottom-left-radius: 8px;
+	border-bottom-right-radius: 8px;
+	margin-bottom: 20px;
+}
+
+.footer {
+    clear: both;
+    height: 30px;
+}
+
+.master_detail_table {
+	display: table;
+	margin-top: 10px;
+	margin-left: 12px;
+	margin-right: 12px;
+}
+
+.master {
+    display: table-cell;
+    text-align: right;
+    min-width: 200px;
+    font-size: 16pt;
+    padding-bottom: 20px;
+    padding-top: 35px;
+    vertical-align: top;
+}
+
+.detail {
+    display: table-cell;
+  	border: 1px solid #555;
+  	background-color: #181818;
+  	padding-top: 30px;
+  	padding-bottom: 20px;
+  	padding-left: 30px;
+  	padding-right: 30px;
+	border-radius: 5;
+	width: 100%;
+}
+
+.detail_function {
+	white-space: pre-wrap;
+	width: 100%;
+	font-size: 11pt;
+	color: #BBB;
+}
+
+.master_element {
+    padding-top: 6px;
+    padding-bottom: 11px;
+    padding-left: 5px;
+    padding-right: 22px;    
+    font-size: 12pt;
+	/* Make our border overlap the detail, even if we're unselected (so it doesn't jump when selected) */
+	position: relative;
+	left: 1px;
+	border-bottom-style: solid;
+	border-bottom-width: 0px;
+}
+
+.selected_master_elem {
+	border: 1px solid #555;
+	border-right: none;
+	background-color: #181818;
+	
+	border-top-left-radius: 5;
+	border-bottom-left-radius: 5;
+	
+	/* Pad one less than .master_element, to accomodate our border. */
+    padding-top: 5px;
+    padding-bottom: 10px;
+    padding-left: 4px;
+}
+
+.master_element_text {
+    text-decoration: none;
+    padding-bottom: 1px;
+    border-bottom-style: inherit;
+    border-bottom-color: inherit;
+    border-bottom-width: 1px;
+}
+
+.master_element_description {
+    text-decoration: none;
+    padding-top: 15px;
+    font-size: 10pt;
+    border-bottom-style: inherit;
+    border-bottom-color: inherit;
+    border-bottom-width: 1px;
+	display: none;
+}
+
+.selected_master_elem > .master_element_description {
+	display: inline;
+}
+
+/* We have a newline between the label and description; hide it initially, but show it when it's selected */
+.master_element > br { display: none; }
+.selected_master_elem > br { display: inherit; }
+
+/* Set this class to suppress the border bottom on master_element_texts with visible descriptions */
+.master_element_no_border { border-bottom-width: 0 }
+
+.colorpicker_term256 {
+	border: solid #444 1px;
+	border-collapse: collapse;
+}
+
+.colorpicker_modifiers {
+	margin-top: 10px;
+	display:inline-block;
+	margin-left: auto;
+	margin-right: auto;
+	color: #AAA;
+	font-size: smaller;
+}
+
+.colorpicker_modifier_cell {
+	cursor: pointer;
+	display:inline-block;
+	text-align: center;
+	border: groove #333 2px;
+	padding: 5px;
+	margin-top: 5px;
+	margin-left: auto;
+	margin-right: auto;
+}
+
+.modifier_cell_selected {
+	color: #CCC;
+	border-color: #AAA;
+	background-color: #444;
+}
+
+.data_table {
+	table-layout:fixed;
+	color: #CCC;
+	width: 100%;
+	padding-left: 10px;
+	padding-right: 10px;
+}
+
+.data_table_row {
+}
+
+.data_table_cell {
+	padding-top: 5px;
+	padding-bottom: 5px;
+	vertical-align: top;
+	overflow:hidden;
+	border-bottom: #444 dotted 1px;
+	word-wrap: break-word;
+}
+
+.history_text {
+	padding-top: 5px;
+	padding-bottom: 5px;
+	vertical-align: top;
+	overflow:hidden;
+	border-bottom: #444 dotted 1px;
+	word-wrap: break-word;	
+}
+
+.history_delete {
+	width: 20px;
+	border-bottom: #444 dotted 1px;
+}
+
+/* The CSS we apply when a table row is filtered */
+.data_table_row_filtered {
+	display: none;
+}
+
+.no_overflow {
+	text-overflow: ellipsis;
+	white-space: nowrap;
+}
+
+.colorpicker_target {
+	margin: 0 0 -50px 0;
+	position: relative;
+	bottom: 47px;
+	float: left; /* for some reason this makes the cells that it overlaps (before adjusting its bottom) clickable in Safari */
+}
+
+.colorpicker_target_tab {
+	cursor: pointer;
+	color: #555;
+	border: solid 1px #555;
+	padding-top: 5px;
+	padding-bottom: 5px;
+	padding-left: 7px;
+	padding-right: 7px;
+	display: inline-block;
+	background-color: black;
+	margin-right: -2px;
+	min-width: 110px;
+	text-align: center;
+}
+
+.colorpicker_target_selected {
+	background-color: #181818; /* same as #detail */
+	color: white;
+}
+
+.colorpicker_term256_row {
+}
+
+.colorpicker_term256_cell {
+	width: 18px;
+	height: 18px;
+	border: solid black 1px;
+}
+
+.colorpicker_term256_selection_indicator {
+	width: 19px;
+	height: 16px;
+	margin: -2px;
+	border: solid white 3px;
+	position: relative;
+	z-index: 2;
+}
+
+.colorpicker_cell_selected {
+	 border: dashed white 3px;
+	 width: 12px;
+	 height: 12px;
+}
+
+.colorpicker_text_sample, .colorpicker_text_sample_tight {
+	font-size: 12pt;
+	padding: 25px;
+	margin: 5px 20px 25px 20px; /* top right bottom left */
+	cursor: pointer;
+	line-height: 1.8em;
+	border: solid #777 1px;
+	position: relative; /* so that our absolutely positioned elements work */
+}
+
+.colorpicker_text_sample_tight {
+	font-size: 10pt;
+	line-height: 1.2em;
+	margin: 0px 6px;
+	max-width: 220px;
+	padding: 5px;
+	white-space:nowrap;
+	overflow: hidden;
+	text-overflow: clip;
+}
+
+.color_picker_background_cells {
+	position: absolute;
+	right: 0px;
+	top: 0px;
+}
+
+.color_picker_background_cells div {
+	width: 14px;
+	height: 14px;
+	border-style: solid;
+	border-color: #777;
+	border-width: 0 0 1px 1px;  /* top right bottom left */
+	float: left;
+}
+
+.color_scheme_choice_label {
+	margin-left: 10px;
+	margin-bottom: 3px;
+	cursor: pointer;
+	font-size: 12pt;
+	white-space: normal;
+}
+
+.color_scheme_choices_list {
+	overflow-y: hidden; /* makes our height account for floats */
+	padding:  0 10px 15px 10px;  /* top right bottom left */
+}
+
+.color_scheme_choice_container {
+	float: left;
+	padding: 5px;
+}
+
+.fake_cursor {
+	background-color: #999;
+}
+
+.error_msg {
+	color: red;
+	font-size: 12pt;
+	margin-left: 24pt;
+	margin-top: 5pt;
+	margin-bottom: 5pt;
+}
+
+img.delete_icon {
+	width: 20px;
+	height: 20px;
+	cursor: pointer;
+	text-decoration: none;
+	border: none;
+}
+
+#table_filter_container {
+	/* top right bottom left*/
+	padding: 0 10 10 30;
+	text-align: right;
+	position: relative;
+	bottom: 10px;
+}
+
+.filter_text_box {
+	width: 250px;
+	padding: 5 10 5 10;
+	background-color: #888;
+	border: #222 solid 3px;
+	border-radius: 15px;
+	font-size: 12pt;
+	color: white;
+	font-weight: bold;
+}
+
+.text_box_transient {
+	color: #C8C8C8;
+}
+
+.prompt_demo {
+	font-size: 12pt;
+	padding: 10px;
+	margin: 5px 20px 25px; /* top right bottom left */
+	cursor: pointer;
+	line-height: 1.8em;
+	border: solid #777 1px;
+	position: relative; /* so that our absolutely positioned elements work */
+}
+
+.save_button, .prompt_save_button {
+	border-radius: 5px;
+	border: solid rgba(71,71,71,0.5) 1px;
+	padding: 5px 8px;
+	font-size: 10pt;
+	display: inline-block;
+	margin-top: 12px;
+	background-color: rgba(128,128,128,0.2);
+	color: #FFF;
+	cursor: pointer;
+}
+
+.prompt_save_button {
+	background-color: #333;
+	border: solid #525252 1px;
+    color: #ffffff;
+	margin: 2px 20px 25px; /* top right bottom left */
+	font-size: 12pt;
+}
+
+.prompt_demo_choice_label {
+	margin: 25px 20px 5px;
+	font-size: 12pt;
+	white-space: normal;
+}
+
+.prompt_demo_text {
+	white-space: pre;
+	line-height: 170%;
+	padding: 4px 12px;
+	font-size: 14pt;
+	top: 0px;
+	bottom: 0px;
+	vertical-align: middle;
+	display: table-cell;
+	height: 72px; /* this is really the min height */
+}
+
+.prompt_function {
+    display: block;
+  	border: 1px solid #555;
+  	background-color: #181818;
+    margin: 5px 20px 5px;
+    border-radius: 5;
+}
+
+.prompt_function_text {
+	white-space: pre-wrap;
+	padding: 15px 3px;
+    width: 100%;
+    height: 25%;
+    overflow: auto;
+}
+
+.external_link_img {
+	width: 16px;
+	height: 16px;
+	vertical-align: text-top;
+	margin-left: 10px;
+};

share/tools/web_config/js/app.js

@@ -0,0 +1,69 @@
+fishconfig = angular.module("fishconfig", ["filters", "controllers"]);
+
+fishconfig.config(
+    ["$routeProvider", function($routeProvider) {
+        $routeProvider
+        .when("/colors", {
+            controller: "colorsController",
+            templateUrl: "partials/colors.html"
+        })
+        .when("/prompt", {
+            controller: "promptController",
+            templateUrl: "partials/prompt.html"
+        })
+        .when("/functions", {
+            controller: "functionsController",
+            templateUrl: "partials/functions.html"
+        })
+        .when("/variables", {
+            controller: "variablesController",
+            templateUrl: "partials/variables.html"
+        })
+        .when("/history", {
+            controller: "historyController",
+            templateUrl: "partials/history.html"
+        })
+        .when("/bindings", {
+            controller: "bindingsController",
+            templateUrl: "partials/bindings.html"
+        })
+        .otherwise({
+            redirectTo: "/colors"
+        })
+    }]);
+
+/* Inspired from http://blog.tomaka17.com/2012/12/random-tricks-when-using-angularjs/ */
+fishconfig.config(function($httpProvider, $compileProvider) {
+    var global_error_element = null;
+
+    var showMessage = function(content) {
+        global_error_element.text(content); 
+    };
+    
+    $httpProvider.responseInterceptors.push(function($q) {
+        return function(promise) {
+            return promise.then(function(successResponse) {
+                showMessage('');
+                return successResponse;
+            }, function(errorResponse) {
+                switch (errorResponse.status) {
+                    case 0:
+                        showMessage("The request received an error. Perhaps the server has shut down.");
+                        break;
+                   case 500:
+                        showMessage('Server internal error: ' + errorResponse.data);
+                        break;
+                    default:
+                        showMessage('Error ' + errorResponse.status + ': ' + errorResponse.data);
+                }
+                return $q.reject(errorResponse);
+            });
+        };
+    });
+
+    $compileProvider.directive('errorMessage', function() {
+        return {
+            link: function(scope, element, attrs) { global_error_element = element; }
+        };
+    });
+});

share/tools/web_config/js/colorutils.js

@@ -0,0 +1,624 @@
+/* TODO: Write an angularjs service to wrap these methods */
+
+term_256_colors = [ //247
+"ffd7d7",
+"d7afaf",
+"af8787",
+"875f5f",
+"5f0000",
+"870000",
+"af0000",
+"d70000",
+"ff0000",
+"ff5f5f",
+"d75f5f",
+"d78787",
+"ff8787",
+"ffafaf",
+"ffaf87",
+"ffaf5f",
+"ffaf00",
+"ff875f",
+"ff8700",
+"ff5f00",
+"d75f00",
+"af5f5f",
+"af5f00",
+"d78700",
+"d7875f",
+"af875f",
+"af8700",
+"875f00",
+"d7af87",
+"ffd7af",
+"ffd787",
+"ffd75f",
+"d7af00",
+"d7af5f",
+"ffd700",
+"ffff5f",
+"ffff00",
+"ffff87",
+"ffffaf",
+"ffffd7",
+"d7ff00",
+"afd75f",
+"d7d700",
+"d7d787",
+"d7d7af",
+"afaf87",
+"87875f",
+"5f5f00",
+"878700",
+"afaf00",
+"afaf5f",
+"d7d75f",
+"d7ff5f",
+"d7ff87",
+"87ff00",
+"afff00",
+"afff5f",
+"afd700",
+"87d700",
+"87af00",
+"5f8700",
+"87af5f",
+"5faf00",
+"afd787",
+"d7ffd7",
+"d7ffaf",
+"afffaf",
+"afff87",
+"5fff00",
+"5fd700",
+"87d75f",
+"5fd75f",
+"87ff5f",
+"5fff5f",
+"87ff87",
+"afd7af",
+"87d787",
+"87d7af",
+"87af87",
+"5f875f",
+"5faf5f",
+"005f00",
+"008700",
+"00af00",
+"00d700",
+"00ff00",
+"00ff5f",
+"5fff87",
+"00ff87",
+"87ffaf",
+"afffd7",
+"5fd787",
+"00d75f",
+"5faf87",
+"00af5f",
+"5fffaf",
+"00ffaf",
+"5fd7af",
+"00d787",
+"00875f",
+"00af87",
+"00d7af",
+"5fffd7",
+"87ffd7",
+"00ffd7",
+"d7ffff",
+"afd7d7",
+"87afaf",
+"5f8787",
+"5fafaf",
+"87d7d7",
+"5fd7d7",
+"5fffff",
+"00ffff",
+"87ffff",
+"afffff",
+"00d7d7",
+"00d7ff",
+"5fd7ff",
+"5fafd7",
+"00afd7",
+"00afff",
+"0087af",
+"00afaf",
+"008787",
+"005f5f",
+"005f87",
+"0087d7",
+"0087ff",
+"5fafff",
+"87afff",
+"5f87d7",
+"5f87ff",
+"005fd7",
+"005fff",
+"005faf",
+"5f87af",
+"87afd7",
+"afd7ff",
+"87d7ff",
+"d7d7ff",
+"afafd7",
+"8787af",
+"afafff",
+"8787d7",
+"8787ff",
+"5f5fff",
+"5f5fd7",
+"5f5faf",
+"5f5f87",
+"00005f",
+"000087",
+"0000af",
+"0000d7",
+"0000ff",
+"5f00ff",
+"5f00d7",
+"5f00af",
+"5f0087",
+"8700af",
+"8700d7",
+"8700ff",
+"af00ff",
+"af00d7",
+"d700ff",
+"d75fff",
+"d787ff",
+"ffafd7",
+"ffafff",
+"ffd7ff",
+"d7afff",
+"d7afd7",
+"af87af",
+"af87d7",
+"af87ff",
+"875fd7",
+"875faf",
+"875fff",
+"af5fff",
+"af5fd7",
+"af5faf",
+"d75fd7",
+"d787d7",
+"ff87ff",
+"ff5fff",
+"ff5fd7",
+"ff00ff",
+"ff00af",
+"ff00d7",
+"d700af",
+"d700d7",
+"af00af",
+"870087",
+"5f005f",
+"87005f",
+"af005f",
+"af0087",
+"d70087",
+"d7005f",
+"ff0087",
+"ff005f",
+"ff5f87",
+"d75f87",
+"d75faf",
+"ff5faf",
+"ff87af",
+"ff87d7",
+"d787af",
+"af5f87",
+"875f87",
+"000000",
+"080808",
+"121212",
+"1c1c1c",
+"262626",
+"303030",
+"3a3a3a",
+"444444",
+"4e4e4e",
+"585858",
+"5f5f5f",
+"626262",
+"6c6c6c",
+"767676",
+"808080",
+"878787",
+"8a8a8a",
+"949494",
+"9e9e9e",
+"a8a8a8",
+"afafaf",
+"b2b2b2",
+"bcbcbc",
+"c6c6c6",
+"d0d0d0",
+"d7d7d7",
+"dadada",
+"e4e4e4",
+"eeeeee",
+"ffffff",
+]
+
+/* Returns array of values from a dictionary (or any object) */
+function dict_values(dict) {
+    var result = [];
+    for (var i in dict) result.push(dict[i]);
+    return result;
+}
+
+/* Return the array of colors as an array of N arrays of length items_per_row */
+function get_colors_as_nested_array(colors, items_per_row) {
+    var result = new Array();
+    for (var idx = 0; idx < colors.length; idx += items_per_row) {
+        var row = new Array();
+        for (var subidx = 0; subidx < items_per_row && idx  + subidx < colors.length; subidx++) {
+            row.push(colors[idx + subidx]);
+        }
+        result.push(row);
+    }
+    return result;
+}
+
+/* Given an RGB color as a hex string, like FF0033, convert to HSL, apply the function to adjust its lightness, then return the new color as an RGB string */
+function adjust_lightness(color_str, func) {
+    /* Hack to handle for example F00 */
+    if (color_str.length == 3) {
+        color_str = color_str[0] + color_str[0] + color_str[1] + color_str[1] + color_str[2] + color_str[2]
+    }
+
+    /* More hacks */
+    if (color_str == 'black') color_str = '000000';
+    if (color_str == 'white') color_str = 'FFFFFF';
+
+
+    rgb = parseInt(color_str, 16)
+    r = (rgb >> 16) & 0xFF
+    g = (rgb >> 8) & 0xFF
+    b = (rgb >> 0) & 0xFF
+
+    hsl = rgb_to_hsl(r, g, b)
+    new_lightness = func(hsl[2])
+    function to_int_str(val) {
+        str = Math.round(val).toString(16)
+        while (str.length < 2)
+            str = '0' + str
+        return str
+    }
+
+    new_rgb = hsl_to_rgb(hsl[0], hsl[1], new_lightness)
+    return to_int_str(new_rgb[0]) + to_int_str(new_rgb[1]) + to_int_str(new_rgb[2])
+}
+
+/* Given a color, compute a "border color" for it that can show it selected */
+function border_color_for_color(color_str) {
+    return adjust_lightness(color_str, function(lightness){
+        var adjust = .5
+        var new_lightness = lightness + adjust
+        if (new_lightness > 1.0 || new_lightness < 0.0) {
+            new_lightness -= 2 * adjust
+        }
+        return new_lightness
+    })
+}
+
+/* Use this function to make a color that contrasts well with the given color */
+function text_color_for_color(color_str) {
+    var adjust = .5
+    function compute_constrast(lightness){
+        var new_lightness = lightness + adjust
+        if (new_lightness > 1.0 || new_lightness < 0.0) {
+            new_lightness -= 2 * adjust
+        }
+        return new_lightness
+    }
+    return adjust_lightness(color_str, compute_constrast);
+}		
+
+function rgb_to_hsl(r, g, b){	
+    r /= 255, g /= 255, b /= 255;
+    var max = Math.max(r, g, b), min = Math.min(r, g, b);
+    var h, s, l = (max + min) / 2;
+
+    if(max == min){
+        h = s = 0; // achromatic
+    }else{
+        var d = max - min;
+        s = l > 0.5 ? d / (2 - max - min) : d / (max + min);
+        switch(max){
+            case r: h = (g - b) / d + (g < b ? 6 : 0); break;
+            case g: h = (b - r) / d + 2; break;
+            case b: h = (r - g) / d + 4; break;
+        }
+        h /= 6;
+    }
+
+    return [h, s, l];
+}
+
+function hsl_to_rgb(h, s, l){
+    var r, g, b;
+
+    if(s == 0){
+        r = g = b = l; // achromatic
+    }else{
+        function hue2rgb(p, q, t){
+            if(t < 0) t += 1;
+            if(t > 1) t -= 1;
+            if(t < 1/6) return p + (q - p) * 6 * t;
+            if(t < 1/2) return q;
+            if(t < 2/3) return p + (q - p) * (2/3 - t) * 6;
+            return p;
+        }
+
+        var q = l < 0.5 ? l * (1 + s) : l + s - l * s;
+        var p = 2 * l - q;
+        r = hue2rgb(p, q, h + 1.0/3);
+        g = hue2rgb(p, q, h);
+        b = hue2rgb(p, q, h - 1.0/3);
+    }
+
+    return [r * 255, g * 255, b * 255]
+}
+
+
+/* Given a color, compute the master text color for it, by giving it a minimum brightness */
+function master_color_for_color(color_str) {
+	return adjust_lightness(color_str, function(lightness){
+		if (lightness < .33) {
+			lightness = .33
+		}
+		return lightness
+	})
+}
+
+/* Given a color name, like 'normal' or 'red' or 'FF00F0', return an RGB color string (or empty string) */
+function interpret_color(str) {
+	str = str.toLowerCase()
+	if (str == 'black') return '000000'
+	if (str == 'red') return 'FF0000'
+	if (str == 'green') return '00FF00'
+	if (str == 'brown') return '725000'
+	if (str == 'yellow') return 'FFFF00'
+	if (str == 'blue') return '0000FF'
+	if (str == 'magenta') return 'FF00FF'
+	if (str == 'purple') return 'FF00FF'
+	if (str == 'cyan') return '00FFFF'
+	if (str == 'white') return 'FFFFFF'
+	if (str == 'normal') return ''
+	return str
+}
+
+var color_scheme_fish_default = {
+    "name": "fish default",
+    "colors": [],
+    'preferred_background': 'black',
+    
+    'autosuggestion': '555',
+    'command': '005fd7',
+    'param': '00afff',
+    'redirection': '00afff',
+    'comment': '990000',
+    'error': 'ff0000',
+    'escape': '00a6b2',
+    'operator': '00a6b2',
+    'quote': '999900',
+    'end': '009900'
+};
+
+
+var TomorrowTheme = {
+    tomorrow_night: {'Background': '1d1f21', 'Current Line': '282a2e', 'Selection': '373b41', 'Foreground': 'c5c8c6', 'Comment': '969896', 'Red': 'cc6666', 'Orange': 'de935f',  'Yellow': 'f0c674', 'Green': 'b5bd68', 'Aqua': '8abeb7', 'Blue': '81a2be', 'Purple': 'b294bb'
+    },
+    
+    tomorrow: {'Background': 'ffffff', 'Current Line': 'efefef', 'Selection': 'd6d6d6', 'Foreground': '4d4d4c', 'Comment': '8e908c', 'Red': 'c82829', 'Orange': 'f5871f', 'Yellow': 'eab700', 'Green': '718c00', 'Aqua': '3e999f', 'Blue': '4271ae', 'Purple': '8959a8'
+    },
+    
+    tomorrow_night_bright: {'Background': '000000', 'Current Line': '2a2a2a', 'Selection': '424242', 'Foreground': 'eaeaea', 'Comment': '969896', 'Red': 'd54e53', 'Orange': 'e78c45', 'Yellow': 'e7c547', 'Green': 'b9ca4a', 'Aqua': '70c0b1', 'Blue': '7aa6da', 'Purple': 'c397d8'},
+    
+    apply: function(theme, receiver){
+        receiver['autosuggestion'] = theme['Comment']
+        receiver['command'] = theme['Purple']
+        receiver['comment'] = theme['Yellow'] /* we want to use comment for autosuggestions */
+        receiver['end'] = theme['Purple']
+        receiver['error'] = theme['Red']
+        receiver['param'] = theme['Blue']
+        receiver['quote'] = theme['Green']
+        receiver['redirection'] = theme['Aqua']
+        
+        receiver['colors'] = []
+        for (var key in theme) receiver['colors'].push(theme[key])
+    }
+}
+
+
+var solarized = {
+    base03: '002b36', base02: '073642', base01: '586e75', base00: '657b83', base0: '839496', base1: '93a1a1', base2: 'eee8d5', base3: 'fdf6e3', yellow: 'b58900', orange: 'cb4b16', red: 'dc322f', magenta: 'd33682', violet: '6c71c4', blue: '268bd2', cyan: '2aa198', green: '859900'
+};
+
+/* Sample color schemes */
+var color_scheme_solarized_light = {
+    name: "Solarized Light",
+    colors: dict_values(solarized),
+    
+    preferred_background: '#' + solarized.base3,
+    
+    autosuggestion: solarized.base1,
+    command: solarized.base01,
+    comment: solarized.base1,
+    end: solarized.blue,
+    error: solarized.red,
+    param: solarized.base00,
+    quote: solarized.base0,
+    redirection: solarized.violet,
+    
+    url: 'http://ethanschoonover.com/solarized'
+};
+
+var color_scheme_solarized_dark = {
+    name: "Solarized Dark",
+    colors: dict_values(solarized),
+    preferred_background: '#' + solarized.base03,
+    
+    autosuggestion: solarized.base01,
+    command: solarized.base1,
+    comment: solarized.base01,
+    end: solarized.blue,
+    error: solarized.red,
+    param: solarized.base0,
+    quote: solarized.base00,
+    redirection: solarized.violet,
+    
+    url: 'http://ethanschoonover.com/solarized'
+};
+
+var color_scheme_tomorrow = {
+    name: 'Tomorrow',
+    preferred_background: 'white',
+    url: 'https://github.com/chriskempson/tomorrow-theme'
+}
+TomorrowTheme.apply(TomorrowTheme.tomorrow, color_scheme_tomorrow)
+
+var color_scheme_tomorrow_night = {
+    name: 'Tomorrow Night',
+    preferred_background: '#232323',
+    url: 'https://github.com/chriskempson/tomorrow-theme'
+}
+TomorrowTheme.apply(TomorrowTheme.tomorrow_night, color_scheme_tomorrow_night)
+
+var color_scheme_tomorrow_night_bright = {
+    'name': 'Tomorrow Night Bright',
+    'preferred_background': 'black',
+    'url': 'https://github.com/chriskempson/tomorrow-theme',
+
+}
+TomorrowTheme.apply(TomorrowTheme.tomorrow_night_bright, color_scheme_tomorrow_night_bright)
+
+function construct_scheme_analogous(label, background, color_list) {
+    return {
+        name: label,
+        preferred_background: background,
+        colors: color_list,
+        
+        command: color_list[0],
+        quote: color_list[6],
+        param: color_list[5],
+        autosuggestion: color_list[4],
+        
+        error: color_list[9],
+        comment: color_list[12],
+        
+        end: color_list[10],
+        redirection: color_list[11]
+    };
+}
+
+function construct_scheme_triad(label, background, color_list) {
+    return {
+        name: label,
+        preferred_background: background,
+        colors: color_list,
+        
+        command: color_list[0],
+        quote: color_list[2],
+        param: color_list[1],
+        autosuggestion: color_list[3],
+        redirection: color_list[4],
+        
+        error: color_list[8],
+        comment: color_list[10],
+        
+        end: color_list[7],
+    };
+}
+
+function construct_scheme_complementary(label, background, color_list) {
+    return {
+        name: label,
+        preferred_background: background,
+        colors: color_list,
+        
+        command: color_list[0],
+        quote: color_list[4],
+        param: color_list[3],
+        autosuggestion: color_list[2],
+        redirection: color_list[6],
+        
+        error: color_list[5],
+        comment: color_list[8],
+        
+        end: color_list[9],
+    };
+}
+
+function construct_color_scheme_mono(label, background, from_end) {
+    var mono_colors = ['000000', '121212', '1c1c1c', '262626', '303030', '3a3a3a', '444444', '4e4e4e', '585858', '5f5f5f', '626262', '6c6c6c', '767676', '808080', '878787', '8a8a8a', '949494', '9e9e9e', 'a8a8a8', 'afafaf', 'b2b2b2', 'bcbcbc', 'c6c6c6', 'd0d0d0', 'd7d7d7', 'dadada', 'e4e4e4', 'eeeeee', 'ffffff'];
+    
+    if (from_end) mono_colors.reverse();
+    
+    return {
+        name: label,
+        preferred_background: background,
+        colors: mono_colors,
+        
+        autosuggestion: '777777',
+        command: mono_colors[0],
+        comment: mono_colors[7],
+        end: mono_colors[12],
+        error: mono_colors[20],
+        param: mono_colors[4],
+        quote: mono_colors[10],
+        redirection: mono_colors[15]
+    };
+}
+
+var additional_color_schemes = [
+    construct_scheme_analogous('Snow Day', 'white', ['164CC9', '325197', '072D83', '4C7AE4', '7596E4', '4319CC', '4C3499', '260885', '724EE5', '9177E5', '02BDBD', '248E8E', '007B7B', '39DEDE', '65DEDE']),
+    
+    construct_scheme_analogous('Lava', '#232323', ['FF9400', 'BF8330', 'A66000', 'FFAE40', 'FFC473', 'FFC000', 'BF9C30', 'A67D00', 'FFD040', 'FFDD73', 'FF4C00', 'BF5B30', 'A63100', 'FF7940', 'FF9D73']),
+    
+    construct_scheme_analogous('Seaweed', '#232323', ['00BF32', '248F40', '007C21', '38DF64', '64DF85', '04819E', '206676', '015367', '38B2CE', '60B9CE', '8EEB00', '7CB02C', '5C9900', 'ACF53D', 'C0F56E']),
+    
+    construct_scheme_triad('Fairground', '#003', ['0772A1', '225E79', '024A68', '3BA3D0', '63AFD0', 'D9005B', 'A3295C', '8D003B', 'EC3B86', 'EC6AA1', 'FFE100', 'BFAE30', 'A69200', 'FFE840', 'FFEE73']),
+    
+    construct_scheme_complementary('Bay Cruise', 'black', ['009999', '1D7373', '006363', '33CCCC', '5CCCCC', 'FF7400', 'BF7130', 'A64B00', 'FF9640', 'FFB273']),
+
+{
+    'name': 'Old School',
+    'preferred_background': 'black',
+    
+    colors: ['00FF00', '30BE30', '00A400', '44FF44', '7BFF7B', 'FF0000', 'BE3030', 'A40000', 'FF7B7B', '777777', 'CCCCCC'],
+    
+    autosuggestion: '777777',
+    command: '00FF00',
+    comment: '30BE30',
+    end: 'FF7B7B',
+    error: 'A40000',
+    param: '30BE30',
+    quote: '44FF44',
+    redirection: '7BFF7B'
+},
+
+{
+    'name': 'Just a Touch',
+    'preferred_background': 'black',
+    
+    colors: ['B0B0B0', '969696', '789276', 'F4F4F4', 'A0A0F0', '666A80', 'F0F0F0', 'D7D7D7', 'B7B7B7', 'FFA779', 'FAFAFA'],
+    
+    autosuggestion: '9C9C9C',
+    command: 'F4F4F4',
+    comment: 'B0B0B0',
+    end: '969696',
+    error: 'FFA779',
+    param: 'A0A0F0',
+    quote: '666A80',
+    redirection: 'FAFAFA'
+},
+
+construct_color_scheme_mono('Mono Lace', 'white', false),
+construct_color_scheme_mono('Mono Smoke', 'black', true)
+];
+

share/tools/web_config/js/controllers.js

@@ -0,0 +1,232 @@
+controllers = angular.module("controllers", []);
+
+controllers.controller("main", function($scope, $location) {
+    $scope.currentTab = "colors";
+
+    $scope.changeView = function(view) {
+        $location.path(view);
+        $scope.currentTab = view;
+    }
+
+})
+
+controllers.controller("colorsController", function($scope, $http) {
+    $scope.changeSelectedColorScheme= function(newScheme) {
+        $scope.selectedColorScheme = newScheme;
+        if ($scope.selectedColorScheme.preferred_background) {
+            $scope.terminalBackgroundColor = $scope.selectedColorScheme.preferred_background;
+        }
+        $scope.selectedColorSetting = 'command';
+        $scope.colorArraysArray = $scope.getColorArraysArray();
+        //TODO: Save button should be shown only when colors are changed
+        $scope.showSaveButton = true;
+    }
+
+    $scope.changeTerminalBackgroundColor = function(color) {
+        $scope.terminalBackgroundColor = color;
+    }
+
+    $scope.text_color_for_color = function(color) {
+        return text_color_for_color(color);
+    }
+
+    $scope.getColorArraysArray = function() {
+        var result = null;
+        if ( $scope.selectedColorScheme.colors && $scope.selectedColorScheme.colors.length > 0)
+            result = get_colors_as_nested_array($scope.selectedColorScheme.colors, 32);
+        else
+            result =  get_colors_as_nested_array(term_256_colors, 32);
+        return result;
+    }
+
+    $scope.selectColorSetting = function(name) {
+        $scope.selectedColorSetting = name;
+    }
+
+    $scope.changeSelectedTextColor = function(color) {
+        $scope.selectedColorScheme[$scope.selectedColorSetting] = color;
+    }
+
+    $scope.sampleTerminalBackgroundColors = ['white', '#' + solarized.base3, '#300', '#003', '#' + solarized.base03, '#232323', 'black'];
+
+    /* Array of FishColorSchemes */
+    $scope.colorSchemes = [color_scheme_fish_default, color_scheme_solarized_light, color_scheme_solarized_dark, color_scheme_tomorrow, color_scheme_tomorrow_night, color_scheme_tomorrow_night_bright];
+    for (var i=0; i < additional_color_schemes.length; i++)
+        $scope.colorSchemes.push(additional_color_schemes[i])
+
+
+    $scope.getCurrentTheme = function() {
+        $http.get("/colors/").success(function(data, status, headers, config) {
+           var currentScheme = { "name": "Current", "colors":[], "preferred_background": "" };
+            for (var i in data) {
+                currentScheme[data[i].name] = data[i].color;
+            }
+            $scope.colorSchemes.splice(0, 0, currentScheme);
+            $scope.changeSelectedColorScheme(currentScheme);
+     })};
+
+    $scope.setTheme = function() {
+        var settingNames = ["autosuggestion", "command", "param", "redirection", "comment", "error", "quote", "end"];
+        for (name in settingNames) {
+            var postData = "what=" + settingNames[name] + "&color=" + $scope.selectedColorScheme[settingNames[name]] + "&background_color=&bold=&underline=";
+            $http.post("/set_color/", postData, { headers: {'Content-Type': 'application/x-www-form-urlencoded'} }).success(function(data, status, headers, config) {
+            })
+        }
+    };
+
+    $scope.getCurrentTheme();
+});
+
+controllers.controller("promptController", function($scope, $http) {
+    $scope.selectedPrompt = null;
+
+    $scope.fetchSamplePrompts= function() {
+        $http.get("/sample_prompts/").success(function(data, status, headers, config) {
+            $scope.samplePrompts = data;
+            $scope.samplePromptsArrayArray = get_colors_as_nested_array($scope.samplePrompts, 1);
+
+            if ($scope.selectedPrompt == null) {
+                $scope.selectPrompt($scope.samplePrompts[0]);
+            }
+        })};
+
+    $scope.selectPrompt = function(promptt) {
+        $scope.selectedPrompt= promptt;
+    }
+
+    $scope.setNewPrompt = function(selectedPrompt) {
+        $http.post("/set_prompt/","what=" + encodeURIComponent(selectedPrompt.function), { headers: {'Content-Type': 'application/x-www-form-urlencoded'} }).success(function(data, status, headers, config){
+
+            // Update attributes of current prompt and select it
+            $scope.samplePrompts[0].demo = selectedPrompt.demo;
+            $scope.samplePrompts[0].function = selectedPrompt.function;
+            $scope.samplePrompts[0].font_size = selectedPrompt.font_size;
+            $scope.selectedPrompt = $scope.samplePrompts[0];
+        })};
+
+    $scope.fetchSamplePrompts();
+});
+
+controllers.controller("functionsController", function($scope, $http) {
+    $scope.selectedFunction = null;
+    $scope.functionDefinition = ""; 
+
+    $scope.selectFunction = function(fun) {
+        $scope.selectedFunction = fun;
+        $scope.fetchFunctionDefinition($scope.selectedFunction);
+    }
+
+    $scope.fetchFunctions= function() {
+        $http.get("/functions/").success(function(data, status, headers, config) {
+            $scope.functions = data;
+            $scope.selectFunction($scope.functions[0]);
+    })};
+
+    $scope.cleanupFishFunction = function (contents) {
+        /* Replace leading tabs and groups of four spaces at the beginning of a line with two spaces. */
+        lines = contents.split('\n')
+        rx = /^[\t ]+/
+        for (var i=0; i < lines.length; i++) {
+            line = lines[i]
+            /* Get leading tabs and spaces */
+            whitespace_arr = rx.exec(line)
+            if (whitespace_arr) {
+                /* Replace four spaces with two spaces, and tabs with two spaces */
+                var whitespace = whitespace_arr[0]
+                new_whitespace = whitespace.replace(/(    )|(\t)/g, '  ')
+                lines[i] = new_whitespace + line.slice(whitespace.length)
+            }
+        }
+        return lines.join('\n')
+    }
+
+    $scope.fetchFunctionDefinition = function(name) {
+         $http.post("/get_function/","what=" + name, { headers: {'Content-Type': 'application/x-www-form-urlencoded'} }).success(function(data, status, headers, config) {
+            $scope.functionDefinition = $scope.cleanupFishFunction(data[0]);
+    })};
+
+    $scope.fetchFunctions();
+});
+
+controllers.controller("variablesController", function($scope, $http) {
+    $scope.query = null;
+
+    $scope.fetchVariables= function() {
+        $http.get("/variables/").success(function(data, status, headers, config) {
+        $scope.variables = data;
+    })};
+
+    $scope.fetchVariables();
+});
+
+controllers.controller("historyController", function($scope, $http, $timeout) {
+    $scope.historyItems = [];
+    $scope.historySize = 0;
+    // Stores items which are yet to be added in history items
+    $scope.remainingItems = [];
+    $scope.selectedItems = [];
+
+    // Populate history items in parts
+    $scope.loadHistory = function() {
+        if ($scope.remainingItems.length <= 0) {
+            $scope.loadingText = "";
+            return;
+        }
+
+        var toLoad = $scope.remainingItems.splice(0, 100);
+        for (i in toLoad) {
+            $scope.historyItems.push(toLoad[i]);
+        }
+
+        $scope.loadingText = "Loading " + $scope.historyItems.length + "/" + $scope.historySize;
+        $timeout($scope.loadHistory, 100);
+    }
+
+    $scope.selectItem = function(item) {
+        var index = $scope.selectedItems.indexOf(item);
+        if ( index >= 0) {
+            $scope.selectedItems.splice(index,1);
+        }
+        else {
+            $scope.selectedItems.push(item);
+        }
+    }
+    // Get history from server
+    $scope.fetchHistory = function() {
+        $http.get("/history/").success(function(data, status, headers, config) {
+        $scope.historySize = data.length;
+        $scope.remainingItems = data;
+
+        /* Call this function 10 times/second */
+        $timeout($scope.loadHistory, 100);
+    })};
+
+    $scope.deleteHistoryItem = function(item) {
+        index = $scope.historyItems.indexOf(item);
+        $http.post("/delete_history_item/","what=" + encodeURIComponent(item), { headers: {'Content-Type': 'application/x-www-form-urlencoded'} }).success(function(data, status, headers, config) {
+        $scope.historyItems.splice(index, 1);
+    })};
+
+    var queryInputTimeout = null;
+    $scope.$watch("queryInput", function() {
+        if (queryInputTimeout){
+            $timeout.cancel(queryInputTimeout);
+        }
+
+        queryInputTimeout = $timeout(function() {
+            $scope.query = $scope.queryInput;
+        }, 1000);
+    });
+
+    $scope.fetchHistory();
+});
+
+controllers.controller("bindingsController", function($scope, $http) {
+    $scope.bindings = [];
+    $scope.fetchBindings = function() {
+        $http.get("/bindings/").success(function(data, status, headers, config) {
+            $scope.bindings = data;
+    })};
+
+    $scope.fetchBindings();
+});

share/tools/web_config/js/filters.js

@@ -0,0 +1,35 @@
+filters = angular.module("filters", []);
+
+filters.filter("filterVariable", function() {
+    return function(variables, query) {
+        var result = []
+        if (variables == undefined) return result;
+        if (query == null) { return variables };
+
+        for(i=0; i<variables.length; ++i) {
+            variable = variables[i];
+            if (variable.name.indexOf(query) != -1 ||  variable.value.indexOf(query) != -1) {
+                result.push(variable);
+            }
+        }
+
+        return result;
+    }
+});
+
+filters.filter("filterBinding", function() {
+    return function(bindings, query) {
+        var result = []
+        if (bindings == undefined) return result;
+        if (query == null) { return bindings};
+
+        for(i=0; i<bindings.length; ++i) {
+            binding = bindings[i];
+            if (binding.command.indexOf(query) != -1 || binding.readable_binding.toLowerCase().indexOf(query.toLowerCase()) != -1) {
+                result.push(binding);
+            }
+        }
+
+        return result;
+    }
+});

share/tools/web_config/partials/bindings.html

@@ -0,0 +1,13 @@
+<div id="table_filter_container" style="display: block;">
+    <input id="table_filter_text_box" class="filter_text_box text_box_transient" placeholder="Filter" ng-model="query">
+</div>
+
+<table class="data_table">
+    <tbody>
+    <tr class="data_table_row" ng-repeat="binding in bindings | filterBinding:query">
+        <td ng-class="{ data_table_cell: true, no_overflow: !binding._is_selected }" style="text-align: right; padding-right: 30px;" ng-click="binding._is_selected = !binding._is_selected">{{ binding.command }}</td>
+        <!-- Some bindings are listed multiple times for e.g. function backward-char is bound to \e\[D as well as key left. Users may want to know why some bindings are listed twice, so the actual binding is shown in next line on a click -->
+        <td ng-class="{ data_table_cell: true, no_overflow: !binding._is_selected }" style="text-align: left; padding-right: 30px;" ng-click="binding._is_selected = !binding._is_selected">{{ binding.readable_binding }} <div ng-show="binding._is_selected"> {{ binding.binding }} </div> </td>
+    </tr>
+    </tbody>
+</table>

share/tools/web_config/partials/colors.html

@@ -0,0 +1,65 @@
+<div>
+    <!-- ko with: color_picker -->
+    <span style="padding-left: 25px">Click to customize each color: </span><br>
+    <div class="colorpicker_text_sample" ng-style="{'background-color': terminalBackgroundColor}">
+        <span style="position: absolute; left: 10px; top: -6px;" data-ng-style="{'color': text_color_for_color(selectedColorScheme.preferred_background || 'white')}">{{ selectedColorScheme.name }}</span><br>
+        <div class="color_picker_background_cells">
+            <div ng-style="{'background-color': color}" ng-repeat="color in sampleTerminalBackgroundColors" ng-click="changeTerminalBackgroundColor(color)"></div>
+        </div>
+        <!-- This is the sample text -->
+        <span data-ng-style="{ 'color': selectedColorScheme.command}" ng-click="selectColorSetting('command')">/bright/vixens</span>
+        <span data-ng-style="{ 'color': selectedColorScheme.param}" ng-click="selectColorSetting('param')">jump</span>
+        <span data-ng-style="{ 'color': selectedColorScheme.end}" ng-click="selectColorSetting('end')">|</span>
+        <span data-ng-style="{ 'color': selectedColorScheme.command}" ng-click="selectColorSetting('command')">dozy</span>
+        <span data-ng-style="{ 'color': selectedColorScheme.quote}" ng-click="selectColorSetting('quote')"> "fowl" </span>
+        <span data-ng-style="{ 'color': selectedColorScheme.redirection}" ng-click="selectColorSetting('redirection')">&gt; quack</span>
+        <span data-ng-style="{ 'color': selectedColorScheme.end}" ng-click="selectColorSetting('end')">&</span>
+        <br>
+        <span data-ng-style="{ 'color': selectedColorScheme.command}" ng-click="selectColorSetting('command')">echo</span>
+        <span data-ng-style="{ 'color': selectedColorScheme.error}" ng-click="selectColorSetting('error')">'Errors are the portals to discovery</span>
+        <br>
+        <span data-ng-style="{ 'color': selectedColorScheme.comment}" ng-click="selectColorSetting('comment')"># This is a comment</span>
+        <br>
+        <span data-ng-style="{ 'color': selectedColorScheme.command}" ng-click="selectColorSetting('command')">Th</span><span data-ng-style="{ 'color': selectedColorScheme.autosuggestion }" ng-click="selectColorSetting('autosuggestion')"><span class="fake_cursor"><span style="visibility: hidden">i</span></span>s is an autosuggestion</span>
+
+        <span class="save_button" style="position: absolute; right: 5px; bottom: 5px;" title="Terminal background color is not set automatically on Apply. See your terminal documentation to set its background color." data-ng-style="{'color': text_color_for_color(selectedColorScheme.preferred_background || 'white')}" ng-show="showSaveButton" ng-click="setTheme()">Apply</span>
+
+    </div>
+            
+    <table class="colorpicker_term256" style="margin: 0px 20px;">
+        <tbody>
+            <tr class="colorpicker_term256_row" data-ng-repeat="color_array in colorArraysArray"> 
+                <td class="colorpicker_term256_cell" data-ng-style="{'background-color': color, 'color': color}" ng-click="changeSelectedTextColor(color)" ng-repeat="color in color_array">
+                                <div class="colorpicker_term256_selection_indicator" ng-show="selectedColorScheme[selectedColorSetting] == color" ng-style="{'border-color': border_color_for_color(color)}"</div>
+                </td>
+            </tr>
+        </tbody>
+        <!-- /ko -->
+    </table>
+    
+    <div class="color_scheme_choices_list">
+        <div class="color_scheme_choice_container" data-ng-repeat="colorScheme in colorSchemes" ng-click="changeSelectedColorScheme(colorScheme)">
+            <div class="color_scheme_choice_label">
+                <!-- This click/clickBubble nonsense is so that we can have a separate URL inside a parent with an onClick handler -->
+                <span>{{colorScheme.name }}</span><!--a data-bind="if: $data.url, click: function(){return true;}, clickBubble: false, attr: { href: $data.url}"><img class="external_link_img" src="external_link.png"></a-->
+            </div>
+            <div class="colorpicker_text_sample_tight" data-ng-style="{'background-color': colorScheme.preferred_background}">
+                <span data-ng-style="{'color': colorScheme.command}">/bright/vixens</span>
+                <span data-ng-style="{'color': colorScheme.param}">jump</span>
+                <span data-ng-style="{'color': colorScheme.end}">|</span>
+                <span data-ng-style="{'color': colorScheme.command}">dozy</span>
+                <span data-ng-style="{'color': colorScheme.quote}"> "fowl" </span>
+                <span data-ng-style="{'color': colorScheme.redirection}">&gt; quack</span>
+                <span data-ng-style="{'color': colorScheme.end}">&</span>
+                <br>
+                <span data-ng-style="{'color': colorScheme.command}">echo</span>
+                <span data-ng-style="{'color': colorScheme.error}">'Errors are the portals to discovery</span>
+                <br>
+                <span data-ng-style="{ 'color': colorScheme.comment}"># This is a comment</span>
+                <br>
+                <span data-ng-style="{ 'color': colorScheme.command}">Th</span><span data-ng-style="{ 'color': colorScheme.autosuggestion}"><span class="fake_cursor"><span style="visibility: hidden">i</span></span>s is an autosuggestion</span>
+            </div>
+        </div>
+    </div>
+    <!-- /ko -->
+</div>

share/tools/web_config/partials/functions.html

@@ -0,0 +1,12 @@
+<div class="master_detail_table">
+    <div class="master">
+        <div ng-repeat="func in functions">
+            <div id="master_{{func}}" ng-class="{'master_element': true, 'selected_master_elem': func == selectedFunction }" ng-style="'color: #aaaaaa'" ng-click="selectFunction(func)">
+                <span class="master_element_text" style="font-size: 11pt;">{{ func }}</span>
+            </div>
+        </div>
+    </div>
+    <div class="detail">
+        <div class="detail_function">{{ functionDefinition }}</div>
+    </div>
+</div>

share/tools/web_config/partials/history.html

@@ -0,0 +1,16 @@
+<div id="table_filter_container">
+    <span ng-show="loadingText.length > 0"> {{ loadingText }} </span>
+    <input id="table_filter_text_box" class="filter_text_box text_box_transient" placeholder="Filter" ng-model="queryInput">
+</div>
+<table class="data_table">
+    <tbody>
+        <tr ng-repeat="item in historyItems | filter:query">
+            <td ng-class="{'history_text': true, 'no_overflow': selectedItems.indexOf(item) < 0}" ng-click="selectItem(item)">{{ item }}</td>
+            <td class="history_delete">
+                <a ng-click="deleteHistoryItem(item)">
+                    <img class="delete_icon" src="delete.png">
+                </a>
+            </td>
+        </tr>
+    </tbody>
+</table>

share/tools/web_config/partials/prompt.html

@@ -0,0 +1,10 @@
+<div style="padding: 0 10px 15px;">
+    <div ng-repeat="prompt in samplePrompts">
+        <div class="prompt_demo_choice_label">{{ prompt.name }}</div>
+        <div ng-bind-html-unsafe='prompt.demo' class="prompt_demo" ng-click="selectPrompt(prompt)"></div>
+        <div class="prompt_function" ng-show="selectedPrompt == prompt">
+            <div class="prompt_function_text">{{ prompt.function }}</div>
+        </div>
+        <span class="prompt_save_button" ng-click="setNewPrompt(selectedPrompt)" ng-show="selectedPrompt == prompt && selectedPrompt != samplePrompts[0]">Use</span>
+    </div>
+</div>

share/tools/web_config/partials/variables.html

@@ -0,0 +1,13 @@
+<div id="table_filter_container">
+    <input id="table_filter_text_box" class="filter_text_box text_box_transient" placeholder="Filter" ng-model="query">
+</div>
+
+<table class="data_table">
+    <tbody>
+    <tr class="data_table_row" ng-repeat="variable in variables | filterVariable:query">
+        <td class="data_table_cell no_overflow" style="text-align: right; padding-right: 30px;">{{ variable.name }}</td>
+        <!-- Small hack to select/unselect variables -->
+        <td ng-class="{'data_table_cell': true, 'no_overflow': !variable._is_selected}" style="text-align: left; padding-right: 30px;" ng-click="variable._is_selected=!variable._is_selected">{{ variable.value }}</td>
+    </tr>
+    </tbody>
+</table>

share/tools/web_config/sample_prompts/informative_git.fish

@@ -44,4 +44,5 @@ function fish_prompt --description 'Write out the prompt'
 
 	echo -n '$ '
 
+	set_color normal
 end

share/tools/web_config/webconfig.py

@@ -101,7 +101,7 @@ def parse_color(color_str):
             # Regular color
             color = better_color(color, parse_one_color(comp))
 
-    return [color, background_color, bold, underline]
+    return {"color": color, "background": background_color, "bold": bold, "underline": underline}
 
 def parse_bool(val):
     val = val.lower()
@@ -124,7 +124,7 @@ def get_special_ansi_escapes():
         import curses
         g_special_escapes_dict = {}
         curses.setupterm()
-        
+
         # Helper function to get a value for a tparm
         def get_tparm(key):
             val = None
@@ -132,12 +132,12 @@ def get_special_ansi_escapes():
             if key: val = curses.tparm(key)
             if val: val = val.decode('utf-8')
             return val
-            
+
         # Just a few for now
         g_special_escapes_dict['exit_attribute_mode'] = get_tparm('sgr0')
         g_special_escapes_dict['bold'] = get_tparm('bold')
         g_special_escapes_dict['underline'] = get_tparm('smul')
-        
+
     return g_special_escapes_dict
 
 # Given a known ANSI escape sequence, convert it to HTML and append to the list
@@ -146,12 +146,12 @@ def append_html_for_ansi_escape(full_val, result, span_open):
 
     # Strip off the initial \x1b[ and terminating m
     val = full_val[2:-1]
-    
+
     # Helper function to close a span if it's open
     def close_span():
         if span_open:
             result.append('</span>')
-    
+
     # term256 foreground color
     match = re.match('38;5;(\d+)', val)
     if match is not None:
@@ -159,7 +159,7 @@ def append_html_for_ansi_escape(full_val, result, span_open):
         html_color =  html_color_for_ansi_color_index(int(match.group(1)))
         result.append('<span style="color: ' + html_color + '">')
         return True # span now open
-        
+
     # term8 foreground color
     if val in [str(x) for x in range(30, 38)]:
         close_span()
@@ -172,26 +172,26 @@ def append_html_for_ansi_escape(full_val, result, span_open):
     if full_val == special_escapes['exit_attribute_mode']:
         close_span()
         return False
-    
+
     # We don't handle bold or underline yet
-        
+
     # Do nothing on failure
     return span_open
-    
+
 def strip_ansi(val):
     # Make a half-assed effort to strip ANSI control sequences
     # We assume that all such sequences start with 0x1b and end with m,
     # which catches most cases
     return re.sub("\x1b[^m]*m", '', val)
-        
+
 def ansi_prompt_line_width(val):
     # Given an ANSI prompt, return the length of its longest line, as in the number of characters it takes up
     # Start by stripping off ANSI
     stripped_val = strip_ansi(val)
-    
+
     # Now count the longest line
     return max([len(x) for x in stripped_val.split('\n')])
-    
+
 
 def ansi_to_html(val):
     # Split us up by ANSI escape sequences
@@ -206,13 +206,13 @@ def ansi_to_html(val):
         )                        # End capture
         """, re.VERBOSE)
     separated = reg.split(val)
-    
+
     # We have to HTML escape the text and convert ANSI escapes into HTML
     # Collect it all into this array
     result = []
-    
+
     span_open = False
-    
+
     # Text is at even indexes, escape sequences at odd indexes
     for i in range(len(separated)):
         component = separated[i]
@@ -223,13 +223,13 @@ def ansi_to_html(val):
         else:
             # It's an escape sequence. Close the previous escape.
             span_open = append_html_for_ansi_escape(component, result, span_open)
-    
+
     # Close final escape
     if span_open: result.append('</span>')
-    
+
     # Remove empty elements
     result = [x for x in result if x]
-    
+
     # Clean up empty spans, the nasty way
     idx = len(result) - 1
     while idx >= 1:
@@ -254,7 +254,161 @@ class FishVar:
         flags = []
         if self.universal: flags.append('universal')
         if self.exported: flags.append('exported')
-        return [self.name, self.value, ', '.join(flags)]
+        return {"name": self.name, "value": self.value, "Flags": ', '.join(flags)}
+
+class FishBinding:
+    """A class that represents keyboard binding """
+
+    def __init__(self, command, binding, readable_binding, description=None):
+        self.command =  command
+        self.binding = binding
+        self.readable_binding = readable_binding
+        self.description = description
+
+    def get_json_obj(self):
+        return {"command" : self.command, "binding": self.binding, "readable_binding": self.readable_binding, "description": self.description }
+
+    def get_readable_binding(command):
+        return command
+
+class BindingParser:
+    """ Class to parse codes for bind command """
+
+    #TODO: What does snext and sprevious mean ?
+    readable_keys=  { "dc":"Delete", "npage": "Page Up", "ppage":"Page Down",
+                    "sdc": "Shift Delete", "shome": "Shift Home",
+                    "left": "Left Arrow", "right": "Right Arrow",
+                    "up": "Up Arrow", "down": "Down Arrow",
+                    "sleft": "Shift Left", "sright": "Shift Right"
+                    }
+
+    def set_buffer(self, buffer, is_key=False):
+        """ Sets code to parse """
+
+        self.buffer = buffer
+        self.is_key = is_key
+        self.index = 0
+
+    def get_char(self):
+        """ Gets next character from buffer """
+
+        c = self.buffer[self.index]
+        self.index += 1
+        return c
+
+    def unget_char(self):
+        """ Goes back by one character for parsing """
+
+        self.index -= 1
+
+    def end(self):
+        """ Returns true if reached end of buffer """
+
+        return self.index >= len(self.buffer)
+
+    def parse_control_sequence(self):
+        """ Parses terminal specifiec control sequences """
+
+        result = ''
+        c = self.get_char()
+
+        # \e0 is used to denote start of control sequence
+        if c == 'O':
+            c = self.get_char()
+
+        # \[1\; is start of control sequence
+        if c == '1':
+            self.get_char();c = self.get_char()
+            if c == ";":
+                c = self.get_char()
+
+        # 3 is Alt
+        if c == '3':
+            result += "ALT - "
+            c = self.get_char()
+
+        # 5 is Ctrl
+        if c == '5':
+            result += "CTRL - "
+            c = self.get_char()
+
+        # 9 is Alt
+        if c == '9':
+            result += "ALT - "
+            c = self.get_char()
+
+        if c == 'A':
+            result += 'Up Arrow'
+        elif c == 'B':
+            result += 'Down Arrow'
+        elif c == 'C':
+            result += 'Right Arrow'
+        elif c == 'D':
+            result += "Left Arrow"
+        elif c == 'F':
+            result += "End"
+        elif c == 'H':
+            result += "Home"
+
+        return result
+
+    def get_readable_binding(self):
+        """ Gets a readable representation of binding """
+
+        if self.is_key:
+            try:
+                result = BindingParser.readable_keys[self.buffer]
+            except KeyError:
+                result = self.buffer.title()
+        else:
+            result = self.parse_binding()
+
+        return result
+
+    def parse_binding(self):
+        readable_command = ''
+        result = ''
+        alt = ctrl = False
+
+        while not self.end():
+            c = self.get_char()
+
+            if c == '\\':
+                c = self.get_char()
+                if c == 'e':
+                    d = self.get_char()
+                    if d == 'O':
+                        self.unget_char()
+                        result += self.parse_control_sequence()
+                    elif d == '\\':
+                        if self.get_char() == '[':
+                            result += self.parse_control_sequence()
+                        else:
+                            self.unget_char()
+                            self.unget_char()
+                            alt = True
+                    else:
+                        alt = True
+                        self.unget_char()
+                elif c == 'c':
+                    ctrl = True
+                elif c == 'n':
+                    result += 'Enter'
+                elif c == 't':
+                    result += 'Tab'
+                elif c == 'b':
+                    result += 'Backspace'
+                else:
+                    result += c
+            else:
+                result += c
+        if ctrl:
+            readable_command += 'CTRL - '
+        if alt:
+            readable_command += 'ALT - '
+
+        return readable_command + result
+
 
 class FishConfigHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
 
@@ -314,7 +468,9 @@ class FishConfigHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
             for match in re.finditer(r"^fish_color_(\S+) ?(.*)", line):
                 color_name, color_value = [x.strip() for x in match.group(1, 2)]
                 color_desc = descriptions.get(color_name, '')
-                result.append([color_name, color_desc, parse_color(color_value)])
+                data = { "name": color_name, "description" : color_desc }
+                data.update(parse_color(color_value))
+                result.append(data)
                 remaining.discard(color_name)
 
         # Ensure that we have all the color names we know about, so that if the
@@ -363,6 +519,39 @@ class FishConfigHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
 
         return [vars[key].get_json_obj() for key in sorted(vars.keys(), key=str.lower)]
 
+    def do_get_bindings(self):
+        """ Get key bindings """
+
+        # Running __fish_config_interactive print fish greeting and
+        # loads key bindings
+        greeting, err = run_fish_cmd(' __fish_config_interactive')
+
+        # Load the key bindings and then list them with bind
+        out, err = run_fish_cmd('__fish_config_interactive; bind')
+
+        # Remove fish greeting from output
+        out = out[len(greeting):]
+
+        # Put all the bindings into a list
+        bindings = []
+        binding_parser = BindingParser()
+
+        for line in out.split('\n'):
+            comps = line.split(' ', 2)
+            if len(comps) < 3:
+                continue
+            if comps[1] == '-k':
+                key_name, command = comps[2].split(' ', 1)
+                binding_parser.set_buffer(key_name, True)
+                fish_binding = FishBinding(command=command, binding=key_name, readable_binding=binding_parser.get_readable_binding())
+            else:
+                binding_parser.set_buffer(comps[1])
+                fish_binding = FishBinding(command=comps[2], binding=comps[1], readable_binding=binding_parser.get_readable_binding())
+
+            bindings.append(fish_binding)
+
+        return [ binding.get_json_obj() for binding in bindings ]
+
     def do_get_history(self):
         # Use \x1e ("record separator") to distinguish between history items. The first
         # backslash is so Python passes one backslash to fish
@@ -371,7 +560,6 @@ class FishConfigHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
         if result: result.pop() # Trim off the trailing element
         return result
 
-
     def do_get_color_for_variable(self, name):
         "Return the color with the given name, or the empty string if there is none"
         out, err = run_fish_cmd("echo -n $" + name)
@@ -397,12 +585,12 @@ class FishConfigHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
         # It's really lame that we always return success here
         out, err = run_fish_cmd('builtin history --save --delete -- ' + escape_fish_cmd(history_item_text))
         return True
-    
+
     def do_set_prompt_function(self, prompt_func):
         cmd = prompt_func + '\n' + 'funcsave fish_prompt'
         out, err = run_fish_cmd(cmd)
         return len(err) == 0
-    
+
     def do_get_prompt(self, command_to_run, prompt_function_text):
         # Return the prompt output by the given command
         prompt_demo_ansi, err = run_fish_cmd(command_to_run)
@@ -414,7 +602,7 @@ class FishConfigHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
         # Return the current prompt
         prompt_func, err = run_fish_cmd('functions fish_prompt')
         return self.do_get_prompt('cd "' + initial_wd + '" ; fish_prompt', prompt_func.strip())
-        
+
     def do_get_sample_prompt(self, text):
         # Return the prompt you get from the given text
         cmd = text + "\n cd \"" + initial_wd + "\" \n fish_prompt\n"
@@ -424,7 +612,7 @@ class FishConfigHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
         # Allow us to skip whitespace, etc.
         if not line: return True
         if line.isspace(): return True
-        
+
         # Parse a comment hash like '# name: Classic'
         match = re.match(r"#\s*(\w+?): (.+)", line, re.IGNORECASE)
         if match:
@@ -434,8 +622,8 @@ class FishConfigHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
             return True
         # Skip other hash comments
         return line.startswith('#')
-        
-        
+
+
     def read_one_sample_prompt(self, fd):
         # Read one sample prompt from fd
         function_lines = []
@@ -448,14 +636,16 @@ class FishConfigHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
             # Maybe not we're not parsing hashes, or maybe we already were not
             if not parsing_hashes:
                 function_lines.append(line)
-        result['function'] = ''.join(function_lines).strip()
-        return result        
-        
+        func = ''.join(function_lines).strip()
+        result.update(self.do_get_sample_prompt(func))
+        return result
+
     def do_get_sample_prompts_list(self):
         result = []
         # Start with the "Current" meta-sample
         result.append({'name': 'Current'})
-        
+        result[0].update(self.do_get_current_prompt())
+
         # Read all of the prompts in sample_prompts
         paths = glob.iglob('sample_prompts/*.fish')
         for path in paths:
@@ -467,7 +657,7 @@ class FishConfigHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
                 # Ignore unreadable files, etc
                 pass
         return result
-        
+
     def font_size_for_ansi_prompt(self, prompt_demo_ansi):
         width = ansi_prompt_line_width(prompt_demo_ansi)
         # Pick a font size
@@ -502,6 +692,8 @@ class FishConfigHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
         elif re.match(r"/color/(\w+)/", p):
             name = re.match(r"/color/(\w+)/", p).group(1)
             output = self.do_get_color_for_variable(name)
+        elif p == '/bindings/':
+            output = self.do_get_bindings()
         else:
             return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
 
@@ -578,7 +770,7 @@ class FishConfigHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
     def log_request(self, code='-', size='-'):
         """ Disable request logging """
         pass
-        
+
 # find fish
 fish_bin_dir = os.environ.get('__fish_bin_dir')
 fish_bin_path = None
@@ -599,7 +791,7 @@ if not fish_bin_dir:
 
 else:
     fish_bin_path = os.path.join(fish_bin_dir, 'fish')
-        
+
 if not os.access(fish_bin_path, os.X_OK):
     print("fish could not be executed at path '%s'. Is fish installed correctly?" % fish_bin_path)
     sys.exit(-1)

tests/test1.in

@@ -15,7 +15,7 @@ echo x-{1}
 echo x-{1,2}
 echo foo-{1,2{3,4}}
 
-# Escpaed newlines
+# Escaped newlines
 echo foo\ bar
 echo foo\
 bar
@@ -99,6 +99,12 @@ echo Test 5 $sta
 echo Test redirections
 begin ; echo output ; echo errput 1>&2  ; end 2>&1 | tee /tmp/tee_test.txt ; cat /tmp/tee_test.txt
 
+# Verify that we can pipe something other than stdout
+# The first line should be printed, since we output to stdout but pipe stderr to /dev/null
+# The second line should not be printed, since we output to stderr and pipe it to /dev/null
+begin ; echo is_stdout ; end 2>| cat > /dev/null
+begin ; echo is_stderr 1>&2 ; end 2>| cat > /dev/null
+
 # echo tests
 
 echo 'abc\ndef'

tests/test1.out

@@ -23,6 +23,7 @@ errput
 output
 errput
 output
+is_stdout
 abc\ndef
 abc
 def

tests/test7.in

@@ -20,15 +20,6 @@ case one
 	echo $status
 end
 
-# Test that non-case tokens inside `switch` don't blow away status
-# (why are these even allowed?)
-false
-switch one
-true
-case one
-	echo $status
-end
-
 #test contains -i
 echo test contains -i
 contains -i string a b c string d

tests/test7.out

@@ -3,7 +3,6 @@
 3
 0
 
-1
 1
 test contains -i
 4

tests/test9.in

@@ -35,3 +35,39 @@ emit test3 foo bar
 
 # test empty argument
 emit
+
+echo "Test break and continue"
+# This should output Ping once
+for i in a b c
+    if not contains $i c ; continue ; end
+    echo Ping
+end
+
+# This should output Pong not at all
+for i in a b c
+    if not contains $i c ; break ; end
+    echo Pong
+end
+
+# This should output Foop three times, and Boop not at all
+set i a a a
+while contains $i a
+    set -e i[-1]
+    echo Foop
+    continue
+    echo Boop
+end
+
+# This should output Doop once
+set i a a a
+while contains $i a
+    set -e i[-1]
+    echo Doop
+    break
+    echo Darp
+end
+
+# Test implicit cd. This should do nothing.
+./
+
+false

tests/test9.out

@@ -2,3 +2,9 @@ Testing that builtins can truncate files
 abc
 before:test1
 received event test3 with args: foo bar
+Test break and continue
+Ping
+Foop
+Foop
+Foop
+Doop

tokenizer.cpp

@@ -14,7 +14,7 @@ segments.
 #include <wctype.h>
 #include <string.h>
 #include <unistd.h>
-
+#include <fcntl.h>
 
 #include "fallback.h"
 #include "util.h"
@@ -50,7 +50,7 @@ segments.
 /**
    Error string for when trying to pipe from fd 0
 */
-#define PIPE_ERROR _( L"Can not use fd 0 as pipe output" )
+#define PIPE_ERROR _( L"Cannot use stdin (fd 0) as pipe output" )
 
 /**
    Characters that separate tokens. They are ordered by frequency of occurrence to increase parsing speed.
@@ -64,7 +64,6 @@ static const wchar_t *tok_desc[] =
 {
     N_(L"Tokenizer not yet initialized"),
     N_(L"Tokenizer error"),
-    N_(L"Invalid token"),
     N_(L"String"),
     N_(L"Pipe"),
     N_(L"End of command"),
@@ -77,6 +76,8 @@ static const wchar_t *tok_desc[] =
     N_(L"Comment")
 };
 
+
+
 /**
    Set the latest tokens string to be the specified error message
 */
@@ -95,16 +96,8 @@ int tok_get_error(tokenizer_t *tok)
 
 tokenizer_t::tokenizer_t(const wchar_t *b, tok_flags_t flags) : buff(NULL), orig_buff(NULL), last_type(TOK_NONE), last_pos(0), has_next(false), accept_unfinished(false), show_comments(false), last_quote(0), error(0), squash_errors(false), cached_lineno_offset(0), cached_lineno_count(0)
 {
-
-    /* We can only generate error messages on the main thread due to wgettext() thread safety issues. */
-    if (!(flags & TOK_SQUASH_ERRORS))
-    {
-        ASSERT_IS_MAIN_THREAD();
-    }
-
     CHECK(b,);
 
-
     this->accept_unfinished = !!(flags & TOK_ACCEPT_UNFINISHED);
     this->show_comments = !!(flags & TOK_SHOW_COMMENTS);
     this->squash_errors = !!(flags & TOK_SQUASH_ERRORS);
@@ -435,65 +428,153 @@ static void read_comment(tokenizer_t *tok)
     tok->last_type = TOK_COMMENT;
 }
 
-/**
-   Read a FD redirection.
+
+
+/* Reads a redirection or an "fd pipe" (like 2>|) from a string. Returns how many characters were consumed. If zero, then this string was not a redirection.
+
+   Also returns by reference the redirection mode, and the fd to redirection. If there is overflow, *out_fd is set to -1.
 */
-static void read_redirect(tokenizer_t *tok, int fd)
+static size_t read_redirection_or_fd_pipe(const wchar_t *buff, enum token_type *out_redirection_mode, int *out_fd)
 {
+    bool errored = false;
+    int fd = 0;
     enum token_type redirection_mode = TOK_NONE;
 
-    if ((*tok->buff == L'>') ||
-            (*tok->buff == L'^'))
-    {
-        tok->buff++;
-        if (*tok->buff == *(tok->buff-1))
-        {
-            tok->buff++;
-            redirection_mode = TOK_REDIRECT_APPEND;
-        }
-        else
-        {
-            redirection_mode = TOK_REDIRECT_OUT;
-        }
+    size_t idx = 0;
 
-        if (*tok->buff == L'|')
+    /* Determine the fd. This may be specified as a prefix like '2>...' or it may be implicit like '>' or '^'. Try parsing out a number; if we did not get any digits then infer it from the first character. Watch out for overflow. */
+    long long big_fd = 0;
+    for (; iswdigit(buff[idx]); idx++)
+    {
+        /* Note that it's important we consume all the digits here, even if it overflows. */
+        if (big_fd <= INT_MAX)
+            big_fd = big_fd * 10 + (buff[idx] - L'0');
+    }
+
+    fd = (big_fd > INT_MAX ? -1 : static_cast<int>(big_fd));
+
+    if (idx == 0)
+    {
+        /* We did not find a leading digit, so there's no explicit fd. Infer it from the type */
+        switch (buff[idx])
         {
-            if (fd == 0)
-            {
-                TOK_CALL_ERROR(tok, TOK_OTHER, PIPE_ERROR);
-                return;
-            }
-            tok->buff++;
-            tok->last_token = to_string<int>(fd);
-            tok->last_type = TOK_PIPE;
-            return;
+            case L'>':
+                fd = STDOUT_FILENO;
+                break;
+            case L'<':
+                fd = STDIN_FILENO;
+                break;
+            case L'^':
+                fd = STDERR_FILENO;
+                break;
+            default:
+                errored = true;
+                break;
         }
     }
-    else if (*tok->buff == L'<')
+
+    /* Either way we should have ended on the redirection character itself like '>' */
+    wchar_t redirect_char = buff[idx++]; //note increment of idx
+    if (redirect_char == L'>' || redirect_char == L'^')
+    {
+        redirection_mode = TOK_REDIRECT_OUT;
+        if (buff[idx] == redirect_char)
+        {
+            /* Doubled up like ^^ or >>. That means append */
+            redirection_mode = TOK_REDIRECT_APPEND;
+            idx++;
+        }
+    }
+    else if (redirect_char == L'<')
     {
-        tok->buff++;
         redirection_mode = TOK_REDIRECT_IN;
     }
     else
     {
-        TOK_CALL_ERROR(tok, TOK_OTHER, REDIRECT_ERROR);
+        /* Something else */
+        errored = true;
     }
 
-    tok->last_token = to_string(fd);
+    /* Optional characters like & or ?, or the pipe char | */
+    wchar_t opt_char = buff[idx];
+    if (opt_char == L'&')
+    {
+        redirection_mode = TOK_REDIRECT_FD;
+        idx++;
+    }
+    else if (opt_char == L'?')
+    {
+        redirection_mode = TOK_REDIRECT_NOCLOB;
+        idx++;
+    }
+    else if (opt_char == L'|')
+    {
+        /* So the string looked like '2>|'. This is not a redirection - it's a pipe! That gets handled elsewhere. */
+        redirection_mode = TOK_PIPE;
+        idx++;
+    }
 
-    if (*tok->buff == L'&')
+    /* Don't return valid-looking stuff on error */
+    if (errored)
     {
-        tok->buff++;
-        tok->last_type = TOK_REDIRECT_FD;
+        idx = 0;
+        redirection_mode = TOK_NONE;
     }
-    else if (*tok->buff == L'?')
+
+    /* Return stuff */
+    if (out_redirection_mode != NULL)
+        *out_redirection_mode = redirection_mode;
+    if (out_fd != NULL)
+        *out_fd = fd;
+
+    return idx;
+}
+
+enum token_type redirection_type_for_string(const wcstring &str, int *out_fd)
+{
+    enum token_type mode = TOK_NONE;
+    int fd = 0;
+    read_redirection_or_fd_pipe(str.c_str(), &mode, &fd);
+    /* Redirections only, no pipes */
+    if (mode == TOK_PIPE || fd < 0)
+        mode = TOK_NONE;
+    if (out_fd != NULL)
+        *out_fd = fd;
+    return mode;
+}
+
+int fd_redirected_by_pipe(const wcstring &str)
+{
+    /* Hack for the common case */
+    if (str == L"|")
     {
-        tok->buff++;
-        tok->last_type = TOK_REDIRECT_NOCLOB;
+        return STDOUT_FILENO;
     }
-    else
+
+    enum token_type mode = TOK_NONE;
+    int fd = 0;
+    read_redirection_or_fd_pipe(str.c_str(), &mode, &fd);
+    /* Pipes only */
+    if (mode != TOK_PIPE || fd < 0)
+        fd = -1;
+    return fd;
+}
+
+int oflags_for_redirection_type(enum token_type type)
+{
+    switch (type)
     {
-        tok->last_type = redirection_mode;
+        case TOK_REDIRECT_APPEND:
+            return O_CREAT | O_APPEND | O_WRONLY;
+        case TOK_REDIRECT_OUT:
+            return O_CREAT | O_WRONLY | O_TRUNC;
+        case TOK_REDIRECT_NOCLOB:
+            return O_CREAT | O_EXCL | O_WRONLY;
+        case TOK_REDIRECT_IN:
+            return O_RDONLY;
+
+        default:
+            return -1;
     }
 }
 
@@ -516,7 +597,7 @@ static bool my_iswspace(wchar_t c)
 
 const wchar_t *tok_get_desc(int type)
 {
-    if (type < 0 || (size_t)type >= sizeof(tok_desc))
+    if (type < 0 || (size_t)type >= (sizeof tok_desc / sizeof *tok_desc))
     {
         return _(L"Invalid token type");
     }
@@ -606,36 +687,56 @@ void tok_next(tokenizer_t *tok)
             break;
 
         case L'>':
-            read_redirect(tok, 1);
-            return;
         case L'<':
-            read_redirect(tok, 0);
-            return;
         case L'^':
-            read_redirect(tok, 2);
-            return;
+        {
+            /* There's some duplication with the code in the default case below. The key difference here is that we must never parse these as a string; a failed redirection is an error! */
+            enum token_type mode = TOK_NONE;
+            int fd = -1;
+            size_t consumed = read_redirection_or_fd_pipe(tok->buff, &mode, &fd);
+            if (consumed == 0 || fd < 0)
+            {
+                TOK_CALL_ERROR(tok, TOK_OTHER, REDIRECT_ERROR);
+            }
+            else
+            {
+                tok->buff += consumed;
+                tok->last_type = mode;
+                tok->last_token = to_string(fd);
+            }
+        }
+        break;
 
         default:
         {
+            /* Maybe a redirection like '2>&1', maybe a pipe like 2>|, maybe just a string */
+            size_t consumed = 0;
+            enum token_type mode = TOK_NONE;
+            int fd = -1;
             if (iswdigit(*tok->buff))
-            {
-                const wchar_t *orig = tok->buff;
-                int fd = 0;
-                while (iswdigit(*tok->buff))
-                    fd = (fd*10) + (*(tok->buff++) - L'0');
+                consumed = read_redirection_or_fd_pipe(tok->buff, &mode, &fd);
 
-                switch (*(tok->buff))
+            if (consumed > 0)
+            {
+                /* It looks like a redirection or a pipe. But we don't support piping fd 0. Note that fd 0 may be -1, indicating overflow; but we don't treat that as a tokenizer error. */
+                if (mode == TOK_PIPE && fd == 0)
                 {
-                    case L'^':
-                    case L'>':
-                    case L'<':
-                        read_redirect(tok, fd);
-                        return;
+                    TOK_CALL_ERROR(tok, TOK_OTHER, PIPE_ERROR);
+                }
+                else
+                {
+                    tok->buff += consumed;
+                    tok->last_type = mode;
+                    tok->last_token = to_string(fd);
                 }
-                tok->buff = orig;
             }
-            read_string(tok);
+            else
+            {
+                /* Not a redirection or pipe, so just a stirng */
+                read_string(tok);
+            }
         }
+        break;
 
     }
 
@@ -693,13 +794,19 @@ wcstring tok_first(const wchar_t *str)
     return result;
 }
 
-int tok_get_pos(tokenizer_t *tok)
+int tok_get_pos(const tokenizer_t *tok)
 {
     CHECK(tok, 0);
-
     return (int)tok->last_pos;
 }
 
+size_t tok_get_extent(const tokenizer_t *tok)
+{
+    CHECK(tok, 0);
+    size_t current_pos = tok->buff - tok->orig_buff;
+    return current_pos > tok->last_pos ? current_pos - tok->last_pos : 0;
+}
+
 
 void tok_set_pos(tokenizer_t *tok, int pos)
 {

tokenizer.h

@@ -19,10 +19,9 @@ enum token_type
 {
     TOK_NONE, /**< Tokenizer not yet constructed */
     TOK_ERROR, /**< Error reading token */
-    TOK_INVALID,/**< Invalid token */
     TOK_STRING,/**< String token */
     TOK_PIPE,/**< Pipe token */
-    TOK_END,/**< End token */
+    TOK_END,/**< End token (semicolon or newline, not literal end) */
     TOK_REDIRECT_OUT, /**< redirection token */
     TOK_REDIRECT_APPEND,/**< redirection append token */
     TOK_REDIRECT_IN,/**< input redirection token */
@@ -143,7 +142,10 @@ int tok_has_next(tokenizer_t *tok);
 /**
   Returns the position of the beginning of the current token in the original string
 */
-int tok_get_pos(tokenizer_t *tok);
+int tok_get_pos(const tokenizer_t *tok);
+
+/** Returns the extent of the current token */
+size_t tok_get_extent(const tokenizer_t *tok);
 
 /** Returns the token type after the current one, without adjusting the position. Optionally returns the next string by reference. */
 enum token_type tok_peek_next(tokenizer_t *tok, wcstring *out_next_string);
@@ -185,6 +187,15 @@ const wchar_t *tok_get_desc(int type);
 */
 int tok_get_error(tokenizer_t *tok);
 
+/* Helper function to determine redirection type from a string, or TOK_NONE if the redirection is invalid. Also returns the fd by reference. */
+enum token_type redirection_type_for_string(const wcstring &str, int *out_fd = NULL);
+
+/* Helper function to determine which fd is redirected by a pipe */
+int fd_redirected_by_pipe(const wcstring &str);
+
+/* Helper function to return oflags (as in open(2)) for a redirection type */
+int oflags_for_redirection_type(enum token_type type);
+
 enum move_word_style_t
 {
     move_word_style_punctuation, //stop at punctuation

wutil.cpp

@@ -476,7 +476,7 @@ const wchar_t *wgettext(const wchar_t *in)
     {
         cstring mbs_in = wcs2string(key);
         char *out = fish_gettext(mbs_in.c_str());
-        val = new wcstring(format_string(L"%s", out));
+        val = new wcstring(format_string(L"%s", out)); //note that this writes into the map!
     }
     errno = err;
     return val->c_str();