Merge branch 'master' into 1218_rebase

Conflicts:
	builtin.cpp
	builtin_commandline.cpp
	highlight.cpp
	input.cpp
	input.h
	reader.cpp
	screen.cpp
	screen.h

.gitignore

@@ -20,7 +20,6 @@ doc_src/index.hdr
 po/*.gmo
 fish
 fish_indent
-fish_pager
 fish_tests
 fishd
 mimedb
@@ -35,3 +34,4 @@ tests/*tmp.*
 tests/foo.txt
 FISH-BUILD-VERSION-FILE
 version
+messages.pot

.travis.yml

@@ -0,0 +1,22 @@
+language: cpp
+compiler:
+  - gcc
+  - clang
+before_install:
+  - sudo apt-get update
+install:
+  - sudo apt-get install --no-install-recommends libncurses5-dev gettext doxygen
+script:
+  - autoreconf
+  - ./configure
+  - make
+  - sudo make install
+  - make test
+notifications:
+  irc:
+    channels:
+      - "irc.oftc.net#fish"
+    template:
+      - "%{repository}#%{build_number} (%{commit} on %{branch} by %{author}): %{message} Details at %{build_url}"
+    use_notice: true
+    skip_join: true

Makefile.in

@@ -57,7 +57,6 @@ CXXFLAGS = @CXXFLAGS@ $(MACROS) $(EXTRA_CXXFLAGS)
 LDFLAGS = @LDFLAGS@
 LDFLAGS_FISH = ${LDFLAGS} @LIBS_FISH@ @LDFLAGS_FISH@
 LDFLAGS_FISH_INDENT = ${LDFLAGS} @LIBS_FISH_INDENT@
-LDFLAGS_FISH_PAGER = ${LDFLAGS} @LIBS_FISH_PAGER@
 LDFLAGS_FISHD = ${LDFLAGS} @LIBS_FISHD@
 LDFLAGS_MIMEDB = ${LDFLAGS} @LIBS_MIMEDB@
 
@@ -91,7 +90,8 @@ FISH_OBJS := function.o builtin.o complete.o env.o exec.o expand.o		\
 	env_universal.o env_universal_common.o input_common.o event.o		\
 	signal.o io.o parse_util.o common.o screen.o path.o autoload.o		\
 	parser_keywords.o iothread.o color.o postfork.o	\
-	builtin_test.o parse_tree.o parse_productions.o parse_execution.cpp
+	builtin_test.o parse_tree.o parse_productions.o parse_execution.cpp \
+	pager.cpp utf8.o
 
 FISH_INDENT_OBJS := fish_indent.o print_help.o common.o	\
 parser_keywords.o wutil.o tokenizer.o
@@ -105,15 +105,6 @@ BUILTIN_FILES := builtin_set.cpp builtin_commandline.cpp	\
 	builtin_set_color.cpp builtin_printf.cpp
 
 
-#
-# All objects that the system needs to build fish_pager
-#
-
-FISH_PAGER_OBJS := fish_pager.o output.o wutil.o 		\
-	input_common.o env_universal.o env_universal_common.o common.o	\
-	print_help.o iothread.o color.o
-
-
 #
 # All objects that the system needs to build fish_tests
 #
@@ -126,7 +117,7 @@ FISH_TESTS_OBJS := $(FISH_OBJS) fish_tests.o
 #
 
 FISHD_OBJS := fishd.o env_universal_common.o wutil.o print_help.o	\
-	common.o
+	common.o utf8.o
 
 
 #
@@ -191,7 +182,7 @@ FUNCTIONS_DIR_FILES := $(wildcard share/functions/*.fish)
 # Programs to install
 #
 
-PROGRAMS := fish mimedb fish_pager fishd fish_indent
+PROGRAMS := fish mimedb fishd fish_indent
 
 #
 # Manual pages to install
@@ -238,7 +229,7 @@ FISH-BUILD-VERSION-FILE: FORCE
 -include FISH-BUILD-VERSION-FILE
 CPPFLAGS += -DFISH_BUILD_VERSION=\"$(FISH_BUILD_VERSION)\"
 .PHONY: FORCE
-env.o fish.o fish_indent.o fish_pager.o fishd.o mimedb.o: FISH-BUILD-VERSION-FILE
+env.o fish.o fish_indent.o fishd.o mimedb.o: FISH-BUILD-VERSION-FILE
 
 
 #
@@ -409,9 +400,9 @@ doc.h: $(HDR_FILES)
 messages.pot: *.cpp *.h share/completions/*.fish share/functions/*.fish
 	if test "$(HAVE_GETTEXT)" = 1; then \
 		xgettext -k_ -kN_ *.cpp *.h -o messages.pot; \
-		if xgettext -j -k_ -kN_ -k--description -LShell share/completions/*.fish share/functions/*.fish -o messages.pot; then true; else \
+		if xgettext -j -k_ -kN_ -k--description -LShell --from-code=UTF-8 share/completions/*.fish share/functions/*.fish -o messages.pot; then true; else \
-			echo "Your xgettext version is too old to build the messages.pot file"\
+			echo "Your xgettext version is too old to build the messages.pot file"; \
-			rm messages.pot\
+			rm messages.pot; \
 			false; \
 		fi; \
 	fi
@@ -698,14 +689,6 @@ fish: $(FISH_OBJS) fish.o
 	$(CXX) $(CXXFLAGS) $(FISH_OBJS) fish.o $(LDFLAGS_FISH) -o $@
 
 
-#
-# Build the fish_pager program.
-#
-
-fish_pager: $(FISH_PAGER_OBJS)
-	$(CXX) $(CXXFLAGS) $(FISH_PAGER_OBJS) $(LDFLAGS_FISH_PAGER) -o $@
-
-
 #
 # Build the fishd program.
 #
@@ -793,41 +776,50 @@ clean:
 # DO NOT DELETE THIS LINE -- make depend depends on it.
 
 autoload.o: config.h autoload.h common.h util.h lru.h wutil.h signal.h env.h
-autoload.o: exec.h proc.h io.h
+autoload.o: exec.h proc.h io.h parse_tree.h tokenizer.h parse_constants.h
 builtin.o: config.h signal.h fallback.h util.h wutil.h common.h builtin.h
-builtin.o: io.h function.h event.h complete.h proc.h parser.h reader.h env.h
+builtin.o: io.h function.h event.h complete.h proc.h parse_tree.h tokenizer.h
-builtin.o: wgetopt.h sanity.h tokenizer.h wildcard.h expand.h input_common.h
+builtin.o: parse_constants.h parser.h reader.h highlight.h env.h color.h
-builtin.o: input.h intern.h exec.h highlight.h screen.h color.h parse_util.h
+builtin.o: wgetopt.h sanity.h wildcard.h expand.h input_common.h input.h
-builtin.o: autoload.h lru.h parser_keywords.h path.h history.h
+builtin.o: intern.h exec.h parse_util.h autoload.h lru.h parser_keywords.h
-builtin.o: builtin_set.cpp builtin_commandline.cpp builtin_complete.cpp
+builtin.o: path.h history.h builtin_set.cpp builtin_commandline.cpp
-builtin.o: builtin_ulimit.cpp builtin_jobs.cpp builtin_printf.cpp
+builtin.o: builtin_complete.cpp builtin_ulimit.cpp builtin_jobs.cpp
+builtin.o: builtin_set_color.cpp output.h screen.h builtin_printf.cpp
 builtin_commandline.o: config.h signal.h fallback.h util.h wutil.h common.h
-builtin_commandline.o: builtin.h io.h wgetopt.h reader.h complete.h proc.h
+builtin_commandline.o: builtin.h io.h wgetopt.h reader.h complete.h
-builtin_commandline.o: parser.h event.h function.h tokenizer.h input_common.h
+builtin_commandline.o: highlight.h env.h color.h proc.h parse_tree.h
-builtin_commandline.o: input.h parse_util.h autoload.h lru.h
+builtin_commandline.o: tokenizer.h parse_constants.h parser.h event.h
+builtin_commandline.o: function.h input_common.h input.h parse_util.h
+builtin_commandline.o: autoload.h lru.h
 builtin_complete.o: config.h signal.h fallback.h util.h wutil.h common.h
 builtin_complete.o: builtin.h io.h complete.h wgetopt.h parser.h proc.h
-builtin_complete.o: event.h function.h reader.h
+builtin_complete.o: parse_tree.h tokenizer.h parse_constants.h event.h
+builtin_complete.o: function.h reader.h highlight.h env.h color.h
 builtin_jobs.o: config.h fallback.h signal.h util.h wutil.h common.h
-builtin_jobs.o: builtin.h io.h proc.h parser.h event.h function.h wgetopt.h
+builtin_jobs.o: builtin.h io.h proc.h parse_tree.h tokenizer.h
+builtin_jobs.o: parse_constants.h parser.h event.h function.h wgetopt.h
+builtin_printf.o: common.h util.h
 builtin_set.o: config.h signal.h fallback.h util.h wutil.h common.h builtin.h
-builtin_set.o: io.h env.h expand.h wgetopt.h proc.h parser.h event.h
+builtin_set.o: io.h env.h expand.h wgetopt.h proc.h parse_tree.h tokenizer.h
-builtin_set.o: function.h
+builtin_set.o: parse_constants.h parser.h event.h function.h
+builtin_set_color.o: config.h builtin.h util.h io.h common.h color.h output.h
+builtin_set_color.o: screen.h highlight.h env.h
 builtin_test.o: config.h common.h util.h builtin.h io.h wutil.h proc.h
-builtin_test.o: signal.h
+builtin_test.o: signal.h parse_tree.h tokenizer.h parse_constants.h
 builtin_ulimit.o: config.h fallback.h signal.h util.h builtin.h io.h common.h
 builtin_ulimit.o: wgetopt.h
-builtin_printf.o: wgetopt.h
 color.o: color.h config.h common.h util.h fallback.h signal.h
 common.o: config.h fallback.h signal.h util.h wutil.h common.h expand.h
-common.o: proc.h io.h wildcard.h parser.h event.h function.h complete.h
+common.o: proc.h io.h parse_tree.h tokenizer.h parse_constants.h wildcard.h
-common.o: util.cpp fallback.cpp
+common.o: complete.h parser.h event.h function.h util.cpp fallback.cpp
 complete.o: config.h signal.h fallback.h util.h tokenizer.h common.h
-complete.o: wildcard.h expand.h proc.h io.h parser.h event.h function.h
+complete.o: wildcard.h expand.h complete.h proc.h io.h parse_tree.h
-complete.o: complete.h builtin.h env.h exec.h reader.h history.h wutil.h
+complete.o: parse_constants.h parser.h event.h function.h builtin.h env.h
-complete.o: intern.h parse_util.h autoload.h lru.h parser_keywords.h path.h
+complete.o: exec.h reader.h highlight.h color.h history.h wutil.h intern.h
-env.o: config.h signal.h fallback.h util.h wutil.h common.h proc.h io.h env.h
+complete.o: parse_util.h autoload.h lru.h parser_keywords.h path.h iothread.h
-env.o: sanity.h expand.h history.h reader.h complete.h parser.h event.h
+env.o: config.h signal.h fallback.h util.h wutil.h common.h proc.h io.h
+env.o: parse_tree.h tokenizer.h parse_constants.h env.h sanity.h expand.h
+env.o: history.h reader.h complete.h highlight.h color.h parser.h event.h
 env.o: function.h env_universal.h env_universal_common.h input.h
 env.o: input_common.h path.h
 env_universal.o: config.h signal.h fallback.h util.h common.h wutil.h
@@ -835,89 +827,116 @@ env_universal.o: env_universal_common.h env_universal.h
 env_universal_common.o: config.h signal.h fallback.h util.h common.h wutil.h
 env_universal_common.o: env_universal_common.h
 event.o: config.h signal.h fallback.h util.h wutil.h common.h function.h
-event.o: event.h proc.h io.h parser.h
+event.o: event.h input_common.h proc.h io.h parse_tree.h tokenizer.h
+event.o: parse_constants.h parser.h
 exec.o: config.h signal.h fallback.h util.h iothread.h postfork.h common.h
-exec.o: proc.h io.h wutil.h exec.h parser.h event.h function.h builtin.h
+exec.o: proc.h io.h parse_tree.h tokenizer.h parse_constants.h wutil.h exec.h
-exec.o: env.h wildcard.h expand.h sanity.h parse_util.h autoload.h lru.h
+exec.o: parser.h event.h function.h builtin.h env.h wildcard.h expand.h
+exec.o: complete.h sanity.h parse_util.h autoload.h lru.h
 expand.o: config.h signal.h fallback.h util.h common.h wutil.h env.h proc.h
-expand.o: io.h parser.h event.h function.h expand.h wildcard.h exec.h
+expand.o: io.h parse_tree.h tokenizer.h parse_constants.h parser.h event.h
-expand.o: tokenizer.h complete.h parse_util.h autoload.h lru.h
+expand.o: function.h expand.h wildcard.h complete.h exec.h iothread.h
+expand.o: parse_util.h autoload.h lru.h
 fallback.o: config.h fallback.h signal.h util.h
 fish.o: config.h signal.h fallback.h util.h common.h reader.h io.h complete.h
-fish.o: builtin.h function.h event.h wutil.h env.h sanity.h proc.h parser.h
+fish.o: highlight.h env.h color.h builtin.h function.h event.h wutil.h
-fish.o: expand.h intern.h exec.h output.h screen.h color.h history.h path.h
+fish.o: sanity.h proc.h parse_tree.h tokenizer.h parse_constants.h parser.h
+fish.o: expand.h intern.h exec.h output.h screen.h history.h path.h input.h
+fish.o: input_common.h
 fish_indent.o: config.h fallback.h signal.h util.h common.h wutil.h
 fish_indent.o: tokenizer.h print_help.h parser_keywords.h
-fish_pager.o: config.h signal.h fallback.h util.h wutil.h common.h complete.h
-fish_pager.o: output.h screen.h color.h input_common.h env_universal.h
-fish_pager.o: env_universal_common.h print_help.h
 fish_tests.o: config.h signal.h fallback.h util.h common.h proc.h io.h
-fish_tests.o: reader.h complete.h builtin.h function.h event.h autoload.h
+fish_tests.o: parse_tree.h tokenizer.h parse_constants.h reader.h complete.h
-fish_tests.o: lru.h wutil.h env.h expand.h parser.h tokenizer.h output.h
+fish_tests.o: highlight.h env.h color.h builtin.h function.h event.h
-fish_tests.o: screen.h color.h exec.h path.h history.h highlight.h iothread.h
+fish_tests.o: autoload.h lru.h wutil.h expand.h parser.h output.h screen.h
-fish_tests.o: postfork.h
+fish_tests.o: exec.h path.h history.h iothread.h postfork.h parse_util.h
+fish_tests.o: pager.h
 fishd.o: config.h signal.h fallback.h util.h common.h wutil.h
 fishd.o: env_universal_common.h path.h env.h print_help.h
 function.o: config.h signal.h wutil.h common.h util.h fallback.h function.h
-function.o: event.h proc.h io.h parser.h intern.h reader.h complete.h
+function.o: event.h proc.h io.h parse_tree.h tokenizer.h parse_constants.h
-function.o: parse_util.h autoload.h lru.h parser_keywords.h env.h expand.h
+function.o: parser.h intern.h reader.h complete.h highlight.h env.h color.h
+function.o: parse_util.h autoload.h lru.h parser_keywords.h expand.h
 highlight.o: config.h signal.h fallback.h util.h wutil.h common.h highlight.h
-highlight.o: env.h screen.h color.h tokenizer.h proc.h io.h parser.h event.h
+highlight.o: env.h color.h tokenizer.h proc.h io.h parse_tree.h
-highlight.o: function.h parse_util.h autoload.h lru.h parser_keywords.h
+highlight.o: parse_constants.h parser.h event.h function.h parse_util.h
-highlight.o: builtin.h expand.h sanity.h complete.h output.h wildcard.h
+highlight.o: autoload.h lru.h parser_keywords.h builtin.h expand.h sanity.h
-highlight.o: path.h history.h
+highlight.o: complete.h output.h screen.h wildcard.h path.h history.h
 history.o: config.h fallback.h signal.h util.h sanity.h tokenizer.h common.h
-history.o: wutil.h history.h intern.h path.h env.h autoload.h lru.h
+history.o: reader.h io.h complete.h highlight.h env.h color.h wutil.h
-history.o: iothread.h
+history.o: history.h intern.h path.h autoload.h lru.h iothread.h
 input.o: config.h signal.h fallback.h util.h wutil.h common.h reader.h io.h
-input.o: complete.h proc.h sanity.h input_common.h input.h parser.h event.h
+input.o: complete.h highlight.h env.h color.h proc.h parse_tree.h tokenizer.h
-input.o: function.h env.h expand.h output.h screen.h color.h intern.h
+input.o: parse_constants.h sanity.h input_common.h input.h parser.h event.h
+input.o: function.h expand.h output.h screen.h intern.h
 input_common.o: config.h fallback.h signal.h util.h common.h wutil.h
 input_common.o: input_common.h env_universal.h env_universal_common.h
 input_common.o: iothread.h
 intern.o: config.h fallback.h signal.h util.h wutil.h common.h intern.h
 io.o: config.h fallback.h signal.h util.h wutil.h common.h exec.h proc.h io.h
+io.o: parse_tree.h tokenizer.h parse_constants.h
 iothread.o: config.h iothread.h common.h util.h signal.h
 key_reader.o: config.h common.h util.h fallback.h signal.h input_common.h
 kill.o: config.h signal.h fallback.h util.h wutil.h common.h kill.h proc.h
-kill.o: io.h sanity.h env.h exec.h path.h
+kill.o: io.h parse_tree.h tokenizer.h parse_constants.h sanity.h env.h exec.h
+kill.o: path.h
 mimedb.o: config.h xdgmime.h fallback.h signal.h util.h print_help.h
 output.o: config.h signal.h fallback.h util.h wutil.h common.h expand.h
-output.o: output.h screen.h color.h highlight.h env.h
+output.o: output.h screen.h highlight.h env.h color.h
+pager.o: config.h pager.h complete.h util.h common.h screen.h highlight.h
+pager.o: env.h color.h input_common.h
+parse_execution.o: parse_execution.h config.h util.h parse_tree.h common.h
+parse_execution.o: tokenizer.h parse_constants.h proc.h signal.h io.h
+parse_execution.o: parse_util.h autoload.h lru.h complete.h wildcard.h
+parse_execution.o: expand.h builtin.h parser.h event.h function.h reader.h
+parse_execution.o: highlight.h env.h color.h wutil.h exec.h path.h
+parse_productions.o: parse_productions.h parse_tree.h config.h util.h
+parse_productions.o: common.h tokenizer.h parse_constants.h
+parse_tree.o: parse_productions.h parse_tree.h config.h util.h common.h
+parse_tree.o: tokenizer.h parse_constants.h fallback.h signal.h wutil.h
+parse_tree.o: proc.h io.h
 parse_util.o: config.h fallback.h signal.h util.h wutil.h common.h
-parse_util.o: tokenizer.h parse_util.h autoload.h lru.h expand.h intern.h
+parse_util.o: tokenizer.h parse_util.h autoload.h lru.h parse_tree.h
-parse_util.o: exec.h proc.h io.h env.h wildcard.h
+parse_util.o: parse_constants.h expand.h intern.h exec.h proc.h io.h env.h
+parse_util.o: wildcard.h complete.h parser.h event.h function.h
 parser.o: config.h signal.h fallback.h util.h common.h wutil.h proc.h io.h
-parser.o: parser.h event.h function.h parser_keywords.h tokenizer.h exec.h
+parser.o: parse_tree.h tokenizer.h parse_constants.h parser.h event.h
-parser.o: wildcard.h expand.h builtin.h env.h reader.h complete.h sanity.h
+parser.o: function.h parser_keywords.h exec.h wildcard.h expand.h complete.h
+parser.o: builtin.h env.h reader.h highlight.h color.h sanity.h
 parser.o: env_universal.h env_universal_common.h intern.h parse_util.h
-parser.o: autoload.h lru.h path.h
+parser.o: autoload.h lru.h path.h parse_execution.h
 parser_keywords.o: config.h fallback.h signal.h common.h util.h
 parser_keywords.o: parser_keywords.h
 path.o: config.h fallback.h signal.h util.h common.h env.h wutil.h path.h
 path.o: expand.h
-postfork.o: signal.h postfork.h config.h common.h util.h proc.h io.h wutil.h
+postfork.o: signal.h postfork.h config.h common.h util.h proc.h io.h
-postfork.o: iothread.h exec.h
+postfork.o: parse_tree.h tokenizer.h parse_constants.h wutil.h iothread.h
+postfork.o: exec.h
 print_help.o: print_help.h
 proc.o: config.h signal.h fallback.h util.h wutil.h common.h proc.h io.h
-proc.o: reader.h complete.h sanity.h env.h parser.h event.h function.h
+proc.o: parse_tree.h tokenizer.h parse_constants.h reader.h complete.h
-proc.o: output.h screen.h color.h
+proc.o: highlight.h env.h color.h sanity.h parser.h event.h function.h
+proc.o: output.h screen.h
 reader.o: config.h signal.h fallback.h util.h wutil.h common.h highlight.h
-reader.o: env.h screen.h color.h reader.h io.h complete.h proc.h parser.h
+reader.o: env.h color.h reader.h io.h complete.h proc.h parse_tree.h
-reader.o: event.h function.h history.h sanity.h exec.h expand.h tokenizer.h
+reader.o: tokenizer.h parse_constants.h parser.h event.h function.h history.h
-reader.o: kill.h input_common.h input.h output.h iothread.h intern.h path.h
+reader.o: sanity.h exec.h expand.h kill.h input_common.h input.h output.h
-reader.o: parse_util.h autoload.h lru.h
+reader.o: screen.h iothread.h intern.h path.h parse_util.h autoload.h lru.h
+reader.o: parser_keywords.h pager.h
 sanity.o: config.h signal.h fallback.h util.h common.h sanity.h proc.h io.h
-sanity.o: history.h wutil.h reader.h complete.h kill.h
+sanity.o: parse_tree.h tokenizer.h parse_constants.h history.h wutil.h
+sanity.o: reader.h complete.h highlight.h env.h color.h kill.h
 screen.o: config.h fallback.h signal.h common.h util.h wutil.h output.h
-screen.o: screen.h color.h highlight.h env.h
+screen.o: screen.h highlight.h env.h color.h pager.h complete.h
 signal.o: config.h signal.h common.h util.h fallback.h wutil.h event.h
-signal.o: reader.h io.h complete.h proc.h
+signal.o: reader.h io.h complete.h highlight.h env.h color.h proc.h
+signal.o: parse_tree.h tokenizer.h parse_constants.h
 tokenizer.o: config.h fallback.h signal.h util.h wutil.h common.h tokenizer.h
 util.o: config.h fallback.h signal.h util.h common.h wutil.h
 wgetopt.o: config.h wgetopt.h wutil.h common.h util.h fallback.h signal.h
 wildcard.o: config.h fallback.h signal.h util.h wutil.h common.h complete.h
-wildcard.o: wildcard.h expand.h reader.h io.h exec.h proc.h
+wildcard.o: wildcard.h expand.h reader.h io.h highlight.h env.h color.h
+wildcard.o: exec.h proc.h parse_tree.h tokenizer.h parse_constants.h
 wutil.o: config.h fallback.h signal.h util.h common.h wutil.h
 xdgmime.o: xdgmime.h xdgmimeint.h xdgmimeglob.h xdgmimemagic.h xdgmimealias.h
 xdgmime.o: xdgmimeparent.h

README.md

@@ -1,4 +1,4 @@
-[fish](http://fishshell.com/) - the friendly interactive shell
+[fish](http://fishshell.com/) - the friendly interactive shell [![Build Status](https://travis-ci.org/fish-shell/fish-shell.png?branch=master)](https://travis-ci.org/fish-shell/fish-shell)
 ================================================
 
 fish is a smart and user-friendly command line shell for OS X, Linux, and the rest of the family. fish includes features like syntax highlighting, autosuggest-as-you-type, and fancy tab completions that just work, with no configuration required.
@@ -7,7 +7,7 @@ For more on fish's design philosophy, see the [design document](http://fishshell
 
 ## Quick Start
 
-fish generally works like other shells, like bash or zsh. A few important differences can be found at <http://fishshell.com/tutorial.html> by searching for magic phrase 'unlike other shells'.
+fish generally works like other shells, like bash or zsh. A few important differences can be found at <http://fishshell.com/tutorial.html> by searching for the magic phrase 'unlike other shells'.
 
 Detailed user documentation is available by running `help` within fish, and also at <http://fishshell.com/docs/2.0/index.html>
 
@@ -17,8 +17,12 @@ fish is written in a sane subset of C++98, with a few components from C++TR1. It
 
 fish can be built using autotools or Xcode. autoconf 2.60 or later is required.
 
+fish depends on a curses implementation, such as ncurses. The headers and libraries are required for building.
+
 fish requires gettext for translation support.
 
+Building the documentation requires Doxygen 1.5 or newer.
+
 ### Autotools Build
 
     autoconf
@@ -44,10 +48,22 @@ On Debian or Ubuntu you want:
 
     sudo apt-get install libncurses5-dev
 
-on RedHat, CentOS, or Amazon EC2:
+On RedHat, CentOS, or Amazon EC2:
 
     sudo yum install ncurses-devel
 
+## Runtime Dependencies
+
+fish requires a curses implementation, such as ncurses, to run.
+
+fish requires a number of utilities to operate, which should be present on any Unix, GNU/Linux or OS X system. These include (but are not limited to) hostname, grep, awk, sed, which, and getopt. fish also requires the bc program.
+
+Translation support requires the gettext program.
+
+Some optional features of fish, such as the manual page completion parser and the web configuration tool, require Python.
+
+In order to generate completions from man pages compressed with either lzma or xz, you may need to install an extra Python package. Python versions prior to 2.6 are not supported.  For Python versions 2.6 to 3.2 you need to install the module `backports.lzma`.  How to install it depends on your system and how you installed Python.  Most Linux distributions should include it as a package named `backports-lzma` (or similar).  From version 3.3 onwards, Python already includes the required module.
+
 ## Packages for Linux
 
 Instructions on how to find builds for several Linux distros are at <https://github.com/fish-shell/fish-shell/wiki/Nightly-builds>
@@ -66,12 +82,6 @@ To switch your default shell back, you can run:
 
 Substitute /bin/bash with /bin/tcsh or /bin/zsh as appropriate.
 
-## Optional Dependencies
-
-In order to generate completions from man pages compressed with either lzma or xz, you may need to install an extra Python package.
-
-Python versions prior to 2.6 are not supported.  For Python versions 2.6 to 3.2 you need to install the module `backports.lzma`.  How to install it depends on your system and how you installed Python.  Most Linux distributions should include it as a package named `backports-lzma` (or similar).  From version 3.3 onwards, Python already includes the required module.
-
 ## Contact Us
 
 Questions, comments, rants and raves can be posted to the official fish mailing list at <https://lists.sourceforge.net/lists/listinfo/fish-users> or join us on our IRC channel [#fish at irc.oftc.net](https://webchat.oftc.net/?channels=fish).

autoload.cpp

@@ -19,7 +19,7 @@ static const int kAutoloadStalenessInterval = 15;
 file_access_attempt_t access_file(const wcstring &path, int mode)
 {
     //printf("Touch %ls\n", path.c_str());
-    file_access_attempt_t result = {0};
+    file_access_attempt_t result = {};
     struct stat statbuf;
     if (wstat(path, &statbuf))
     {
@@ -48,9 +48,7 @@ autoload_t::autoload_t(const wcstring &env_var_name_var, const builtin_script_t
     lock(),
     env_var_name(env_var_name_var),
     builtin_scripts(scripts),
-    builtin_script_count(script_count),
+    builtin_script_count(script_count)
-    last_path(),
-    is_loading_set()
 {
     pthread_mutex_init(&lock, NULL);
 }
@@ -94,33 +92,34 @@ int autoload_t::load(const wcstring &cmd, bool reload)
     if (path_var != this->last_path)
     {
         this->last_path = path_var;
+        this->last_path_tokenized.clear();
+        tokenize_variable_array(this->last_path, this->last_path_tokenized);
+        
         scoped_lock locker(lock);
         this->evict_all_nodes();
     }
 
+    /* Mark that we're loading this. Hang onto the iterator for fast erasing later. Note that std::set has guarantees about not invalidating iterators, so this is safe to do across the callouts below.  */
+    typedef std::set<wcstring>::iterator set_iterator_t;
+    std::pair<set_iterator_t, bool> insert_result = is_loading_set.insert(cmd);
+    set_iterator_t where = insert_result.first;
+    bool inserted = insert_result.second;
+
     /** Warn and fail on infinite recursion. It's OK to do this because this function is only called on the main thread. */
-    if (this->is_loading(cmd))
+    if (! inserted)
     {
+        /* We failed to insert */
         debug(0,
               _(L"Could not autoload item '%ls', it is already being autoloaded. "
                 L"This is a circular dependency in the autoloading scripts, please remove it."),
               cmd.c_str());
         return 1;
     }
-
-    /* Mark that we're loading this */
-    is_loading_set.insert(cmd);
-
-    /* Get the list of paths from which we will try to load */
-    std::vector<wcstring> path_list;
-    tokenize_variable_array(path_var, path_list);
-
     /* Try loading it */
-    res = this->locate_file_and_maybe_load_it(cmd, true, reload, path_list);
+    res = this->locate_file_and_maybe_load_it(cmd, true, reload, this->last_path_tokenized);
 
     /* Clean up */
-    bool erased = !! is_loading_set.erase(cmd);
+    is_loading_set.erase(where);
-    assert(erased);
 
     return res;
 }

autoload.h

@@ -65,17 +65,15 @@ private:
     /** The path from which we most recently autoloaded */
     wcstring last_path;
     
+    /** That path, tokenized (split on separators) */
+    wcstring_list_t last_path_tokenized;
+
     /**
        A table containing all the files that are currently being
        loaded. This is here to help prevent recursion.
     */
     std::set<wcstring> is_loading_set;
 
-    bool is_loading(const wcstring &name) const
-    {
-        return is_loading_set.find(name) != is_loading_set.end();
-    }
-
     void remove_all_functions(void)
     {
         this->evict_all_nodes();

build_tools/git_version_gen.sh

@@ -14,7 +14,7 @@ then
 	VN=$(cat version) || VN="$DEF_VER"
 elif test -d .git -o -f .git && type git >/dev/null
 then
-	VN=$(git describe --tags --dirty 2>/dev/null)
+	VN=$(git describe --always --dirty 2>/dev/null)
 else
 	VN="$DEF_VER"
 fi

build_tools/make_tarball.sh

@@ -20,7 +20,7 @@ wd="$PWD"
 prefix="fish"
 
 # Get the version from git-describe
-VERSION=`git describe --tags --dirty 2>/dev/null`
+VERSION=`git describe --dirty 2>/dev/null`
 prefix="$prefix-$VERSION"
 
 # The path where we will output the tar file

builtin.cpp

@@ -576,8 +576,7 @@ static int builtin_bind(parser_t &parser, wchar_t **argv)
         BIND_ERASE,
         BIND_KEY_NAMES,
         BIND_FUNCTION_NAMES
-    }
+    };
-    ;
 
     int argc=builtin_count_args(argv);
     int mode = BIND_INSERT;
@@ -593,46 +592,18 @@ static int builtin_bind(parser_t &parser, wchar_t **argv)
 
     woptind=0;
 
-    static const struct woption
+    static const struct woption long_options[] =
-            long_options[] =
     {
-        {
+        { L"all", no_argument, 0, 'a' },
-            L"all", no_argument, 0, 'a'
+        { L"erase", no_argument, 0, 'e' },
-        }
+        { L"function-names", no_argument, 0, 'f' },
-        ,
+        { L"help", no_argument, 0, 'h' },
-        {
+        { L"key", no_argument, 0, 'k' },
-            L"erase", no_argument, 0, 'e'
+        { L"key-names", no_argument, 0, 'K' },
-        }
+        { L"mode", required_argument, 0, 'M' },
-        ,
+        { L"sets-mode", required_argument, 0, 'm' },
-        {
+        { 0, 0, 0, 0 }
-            L"function-names", no_argument, 0, 'f'
+    };
-        }
-        ,
-        {
-            L"help", no_argument, 0, 'h'
-        }
-        ,
-        {
-            L"key", no_argument, 0, 'k'
-        }
-        ,
-        {
-            L"key-names", no_argument, 0, 'K'
-        }
-        ,
-        {
-            L"mode", required_argument, 0, 'M'
-        }
-        ,
-        {
-            L"sets-mode", required_argument, 0, 'm'
-        }
-        ,
-        {
-            0, 0, 0, 0
-        }
-    }
-    ;
 
     while (1)
     {
@@ -1802,8 +1773,8 @@ static int builtin_pwd(parser_t &parser, wchar_t **argv)
     }
 }
 
-/* This is nearly identical to builtin_function, and is intended to be the successor (with no block manipulation, no function/end split) */
+/** Adds a function to the function set. It calls into function.cpp to perform any heavy lifting. */
-int define_function(parser_t &parser, const wcstring_list_t &c_args, const wcstring &contents, wcstring *out_err)
+int define_function(parser_t &parser, const wcstring_list_t &c_args, const wcstring &contents, int definition_line_offset, wcstring *out_err)
 {
     assert(out_err != NULL);
 
@@ -2101,345 +2072,12 @@ int define_function(parser_t &parser, const wcstring_list_t &c_args, const wcstr
 
         d.definition = contents.c_str();
 
-        // TODO: fix def_offset inside function_add
+        function_add(d, parser, definition_line_offset);
-        function_add(d, parser);
     }
 
     return res;
 }
 
-/**
-   The function builtin, used for providing subroutines.
-   It calls various functions from function.c to perform any heavy lifting.
-*/
-static int builtin_function(parser_t &parser, wchar_t **argv)
-{
-    /* Hack hack hack - with the new parser, this is only invoked for help */
-    if (parser_use_ast())
-    {
-        builtin_print_help(parser, argv[0], stdout_buffer);
-        return STATUS_BUILTIN_OK;
-    }
-
-    int argc = builtin_count_args(argv);
-    int res=STATUS_BUILTIN_OK;
-    wchar_t *desc=0;
-    std::vector<event_t> events;
-    std::auto_ptr<wcstring_list_t> named_arguments(NULL);
-
-    wchar_t *name = 0;
-    bool shadows = true;
-
-    woptind=0;
-
-    function_def_block_t * const fdb = new function_def_block_t();
-    parser.push_block(fdb);
-
-    const struct woption long_options[] =
-    {
-        { L"description", required_argument, 0, 'd' },
-        { L"on-signal", required_argument, 0, 's' },
-        { L"on-job-exit", required_argument, 0, 'j' },
-        { L"on-process-exit", required_argument, 0, 'p' },
-        { L"on-variable", required_argument, 0, 'v' },
-        { L"on-event", required_argument, 0, 'e' },
-        { L"help", no_argument, 0, 'h' },
-        { L"argument-names", no_argument, 0, 'a' },
-        { L"no-scope-shadowing", no_argument, 0, 'S' },
-        { 0, 0, 0, 0 }
-    };
-
-    while (1 && (!res))
-    {
-        int opt_index = 0;
-
-        int opt = wgetopt_long(argc,
-                               argv,
-                               L"d:s:j:p:v:e:haS",
-                               long_options,
-                               &opt_index);
-        if (opt == -1)
-            break;
-
-        switch (opt)
-        {
-            case 0:
-                if (long_options[opt_index].flag != 0)
-                    break;
-                append_format(stderr_buffer,
-                              BUILTIN_ERR_UNKNOWN,
-                              argv[0],
-                              long_options[opt_index].name);
-
-                res = 1;
-                break;
-
-            case 'd':
-                desc=woptarg;
-                break;
-
-            case 's':
-            {
-                int sig = wcs2sig(woptarg);
-
-                if (sig < 0)
-                {
-                    append_format(stderr_buffer,
-                                  _(L"%ls: Unknown signal '%ls'\n"),
-                                  argv[0],
-                                  woptarg);
-                    res=1;
-                    break;
-                }
-                events.push_back(event_t::signal_event(sig));
-                break;
-            }
-
-            case 'v':
-            {
-                if (wcsvarname(woptarg))
-                {
-                    append_format(stderr_buffer,
-                                  _(L"%ls: Invalid variable name '%ls'\n"),
-                                  argv[0],
-                                  woptarg);
-                    res=STATUS_BUILTIN_ERROR;
-                    break;
-                }
-
-                events.push_back(event_t::variable_event(woptarg));
-                break;
-            }
-
-
-            case 'e':
-            {
-                events.push_back(event_t::generic_event(woptarg));
-                break;
-            }
-
-            case 'j':
-            case 'p':
-            {
-                pid_t pid;
-                wchar_t *end;
-                event_t e(EVENT_ANY);
-
-                if ((opt == 'j') &&
-                        (wcscasecmp(woptarg, L"caller") == 0))
-                {
-                    int job_id = -1;
-
-                    if (is_subshell)
-                    {
-                        size_t block_idx = 0;
-
-                        /* Find the outermost substitution block */
-                        for (block_idx = 0; ; block_idx++)
-                        {
-                            const block_t *b = parser.block_at_index(block_idx);
-                            if (b == NULL || b->type() == SUBST)
-                                break;
-                        }
-
-                        /* Go one step beyond that, to get to the caller */
-                        const block_t *caller_block = parser.block_at_index(block_idx + 1);
-                        if (caller_block != NULL && caller_block->job != NULL)
-                        {
-                            job_id = caller_block->job->job_id;
-                        }
-                    }
-
-                    if (job_id == -1)
-                    {
-                        append_format(stderr_buffer,
-                                      _(L"%ls: Cannot find calling job for event handler\n"),
-                                      argv[0]);
-                        res=1;
-                    }
-                    else
-                    {
-                        e.type = EVENT_JOB_ID;
-                        e.param1.job_id = job_id;
-                    }
-
-                }
-                else
-                {
-                    errno = 0;
-                    pid = fish_wcstoi(woptarg, &end, 10);
-                    if (errno || !end || *end)
-                    {
-                        append_format(stderr_buffer,
-                                      _(L"%ls: Invalid process id %ls\n"),
-                                      argv[0],
-                                      woptarg);
-                        res=1;
-                        break;
-                    }
-
-
-                    e.type = EVENT_EXIT;
-                    e.param1.pid = (opt=='j'?-1:1)*abs(pid);
-                }
-                if (res)
-                {
-                    /* nothing */
-                }
-                else
-                {
-                    events.push_back(e);
-                }
-                break;
-            }
-
-            case 'a':
-                if (named_arguments.get() == NULL)
-                    named_arguments.reset(new wcstring_list_t);
-                break;
-
-            case 'S':
-                shadows = 0;
-                break;
-
-            case 'h':
-                parser.pop_block();
-                parser.push_block(new fake_block_t());
-                builtin_print_help(parser, argv[0], stdout_buffer);
-                return STATUS_BUILTIN_OK;
-
-            case '?':
-                builtin_unknown_option(parser, argv[0], argv[woptind-1]);
-                res = 1;
-                break;
-
-        }
-
-    }
-
-    if (!res)
-    {
-
-        if (argc == woptind)
-        {
-            append_format(stderr_buffer,
-                          _(L"%ls: Expected function name\n"),
-                          argv[0]);
-            res=1;
-        }
-        else if (wcsfuncname(argv[woptind]))
-        {
-            append_format(stderr_buffer,
-                          _(L"%ls: Illegal function name '%ls'\n"),
-                          argv[0],
-                          argv[woptind]);
-
-            res=1;
-        }
-        else if (parser_keywords_is_reserved(argv[woptind]))
-        {
-
-            append_format(stderr_buffer,
-                          _(L"%ls: The name '%ls' is reserved,\nand can not be used as a function name\n"),
-                          argv[0],
-                          argv[woptind]);
-
-            res=1;
-        }
-        else if (! wcslen(argv[woptind]))
-        {
-            append_format(stderr_buffer, _(L"%ls: No function name given\n"), argv[0]);
-        }
-        else
-        {
-
-            name = argv[woptind++];
-
-            if (named_arguments.get())
-            {
-                while (woptind < argc)
-                {
-                    if (wcsvarname(argv[woptind]))
-                    {
-                        append_format(stderr_buffer,
-                                      _(L"%ls: Invalid variable name '%ls'\n"),
-                                      argv[0],
-                                      argv[woptind]);
-                        res = STATUS_BUILTIN_ERROR;
-                        break;
-                    }
-
-                    named_arguments->push_back(argv[woptind++]);
-                }
-            }
-            else if (woptind != argc)
-            {
-                append_format(stderr_buffer,
-                              _(L"%ls: Expected one argument, got %d\n"),
-                              argv[0],
-                              argc);
-                res=1;
-
-            }
-        }
-    }
-
-    if (res)
-    {
-        size_t i;
-        size_t chars=0;
-
-        builtin_print_help(parser, argv[0], stderr_buffer);
-        const wchar_t *cfa =  _(L"Current functions are: ");
-        stderr_buffer.append(cfa);
-        chars += wcslen(cfa);
-
-        wcstring_list_t names = function_get_names(0);
-        sort(names.begin(), names.end());
-
-        for (i=0; i<names.size(); i++)
-        {
-            const wchar_t *nxt = names.at(i).c_str();
-            size_t l = wcslen(nxt + 2);
-            if (chars+l > (size_t)common_get_width())
-            {
-                chars = 0;
-                stderr_buffer.push_back(L'\n');
-            }
-
-            stderr_buffer.append(nxt);
-            stderr_buffer.append(L"  ");
-        }
-        stderr_buffer.push_back(L'\n');
-
-        parser.pop_block();
-        parser.push_block(new fake_block_t());
-    }
-    else
-    {
-        function_data_t &d = fdb->function_data;
-
-        d.name = name;
-        if (desc)
-            d.description = desc;
-        d.events.swap(events);
-        d.shadows = shadows;
-        if (named_arguments.get())
-            d.named_arguments.swap(*named_arguments);
-
-        for (size_t i=0; i<d.events.size(); i++)
-        {
-            event_t &e = d.events.at(i);
-            e.function_name = d.name;
-        }
-    }
-
-    parser.current_block()->tok_pos = parser.get_pos();
-    parser.current_block()->skip = 1;
-
-    return STATUS_BUILTIN_OK;
-}
-
 /**
    The random builtin. For generating random numbers.
 */
@@ -3715,263 +3353,6 @@ static int builtin_bg(parser_t &parser, wchar_t **argv)
 }
 
 
-/**
-   Builtin for looping over a list
-*/
-static int builtin_for(parser_t &parser, wchar_t **argv)
-{
-    int argc = builtin_count_args(argv);
-    int res=STATUS_BUILTIN_ERROR;
-
-    /* Hackish - if we have no arguments other than the command, we are a "naked invocation" and we just print help */
-    if (argc == 1)
-    {
-        builtin_print_help(parser, argv[0], stdout_buffer);
-        return STATUS_BUILTIN_ERROR;
-    }
-
-    if (argc < 3)
-    {
-        append_format(stderr_buffer,
-                      BUILTIN_FOR_ERR_COUNT,
-                      argv[0] ,
-                      argc);
-        builtin_print_help(parser, argv[0], stderr_buffer);
-    }
-    else if (wcsvarname(argv[1]))
-    {
-        append_format(stderr_buffer,
-                      BUILTIN_FOR_ERR_NAME,
-                      argv[0],
-                      argv[1]);
-        builtin_print_help(parser, argv[0], stderr_buffer);
-    }
-    else if (wcscmp(argv[2], L"in") != 0)
-    {
-        append_format(stderr_buffer,
-                      BUILTIN_FOR_ERR_IN,
-                      argv[0]);
-        builtin_print_help(parser, argv[0], stderr_buffer);
-    }
-    else
-    {
-        res=0;
-    }
-
-
-    if (res)
-    {
-        parser.push_block(new fake_block_t());
-    }
-    else
-    {
-        const wchar_t *for_variable = argv[1];
-        for_block_t *fb = new for_block_t(for_variable);
-        parser.push_block(fb);
-        fb->tok_pos = parser.get_pos();
-
-        /* Note that we store the sequence of values in opposite order */
-        wcstring_list_t &for_vars = fb->sequence;
-        for (int i=argc-1; i>3; i--)
-            for_vars.push_back(argv[i]);
-
-        if (argc > 3)
-        {
-            env_set(for_variable, argv[3], ENV_LOCAL);
-        }
-        else
-        {
-            parser.current_block()->skip=1;
-        }
-    }
-    return res;
-}
-
-/**
-   The begin builtin. Creates a new block.
-*/
-static int builtin_begin(parser_t &parser, wchar_t **argv)
-{
-    parser.push_block(new scope_block_t(BEGIN));
-    parser.current_block()->tok_pos = parser.get_pos();
-    return proc_get_last_status();
-}
-
-
-/**
-   Builtin for ending a block of code, such as a for-loop or an if statement.
-
-   The end command is whare a lot of the block-level magic happens.
-*/
-static int builtin_end(parser_t &parser, wchar_t **argv)
-{
-    if (! parser.block_at_index(1))
-    {
-        append_format(stderr_buffer,
-                      _(L"%ls: Not inside of block\n"),
-                      argv[0]);
-
-        builtin_print_help(parser, argv[0], stderr_buffer);
-        return STATUS_BUILTIN_ERROR;
-    }
-    else
-    {
-        /**
-           By default, 'end' kills the current block scope. But if we
-           are rewinding a loop, this should be set to false, so that
-           variables in the current loop scope won't die between laps.
-        */
-        bool kill_block = true;
-
-        block_t * const current_block = parser.current_block();
-        switch (current_block->type())
-        {
-            case WHILE:
-            {
-                /*
-                  If this is a while loop, we rewind the loop unless
-                  it's the last lap, in which case we continue.
-                */
-                if (!(current_block->skip && (current_block->loop_status != LOOP_CONTINUE)))
-                {
-                    current_block->loop_status = LOOP_NORMAL;
-                    current_block->skip = 0;
-                    kill_block = false;
-                    parser.set_pos(current_block->tok_pos);
-                    while_block_t *blk = static_cast<while_block_t *>(current_block);
-                    blk->status = WHILE_TEST_AGAIN;
-                }
-
-                break;
-            }
-
-            case IF:
-            case SUBST:
-            case BEGIN:
-            case SWITCH:
-            case FAKE:
-                /*
-                  Nothing special happens at the end of these commands. The scope just ends.
-                */
-
-                break;
-
-            case FOR:
-            {
-                /*
-                  set loop variable to next element, and rewind to the beginning of the block.
-                */
-                for_block_t *fb = static_cast<for_block_t *>(current_block);
-                wcstring_list_t &for_vars = fb->sequence;
-                if (current_block->loop_status == LOOP_BREAK)
-                {
-                    for_vars.clear();
-                }
-
-                if (! for_vars.empty())
-                {
-                    const wcstring val = for_vars.back();
-                    for_vars.pop_back();
-                    const wcstring &for_variable = fb->variable;
-                    env_set(for_variable, val.c_str(),  ENV_LOCAL);
-                    current_block->loop_status = LOOP_NORMAL;
-                    current_block->skip = 0;
-
-                    kill_block = false;
-                    parser.set_pos(current_block->tok_pos);
-                }
-                break;
-            }
-
-            case FUNCTION_DEF:
-            {
-                function_def_block_t *fdb = static_cast<function_def_block_t *>(current_block);
-                function_data_t &d = fdb->function_data;
-
-                if (d.name.empty())
-                {
-                    /* Disallow empty function names */
-                    append_format(stderr_buffer, _(L"%ls: No function name given\n"), argv[0]);
-
-                    /* Return an error via a crummy way. Don't just return here, since we need to pop the block. */
-                    proc_set_last_status(STATUS_BUILTIN_ERROR);
-                }
-                else
-                {
-                    /**
-                       Copy the text from the beginning of the function
-                       until the end command and use as the new definition
-                       for the specified function
-                    */
-
-                    wchar_t *def = wcsndup(parser.get_buffer()+current_block->tok_pos,
-                                           parser.get_job_pos()-current_block->tok_pos);
-                    d.definition = def;
-
-                    function_add(d, parser);
-                    free(def);
-                }
-            }
-            break;
-
-            default:
-                assert(false); //should never get here
-                break;
-
-        }
-        if (kill_block)
-        {
-            parser.pop_block();
-        }
-
-        /*
-          If everything goes ok, return status of last command to execute.
-        */
-        return proc_get_last_status();
-    }
-}
-
-/**
-   Builtin for executing commands if an if statement is false
-*/
-static int builtin_else(parser_t &parser, wchar_t **argv)
-{
-    bool block_ok = false;
-    if_block_t *if_block = NULL;
-    if (parser.current_block() != NULL && parser.current_block()->type() == IF)
-    {
-        if_block = static_cast<if_block_t *>(parser.current_block());
-        /* Ensure that we're past IF but not up to an ELSE */
-        if (if_block->if_expr_evaluated && ! if_block->else_evaluated)
-        {
-            block_ok = true;
-        }
-    }
-
-    if (! block_ok)
-    {
-        append_format(stderr_buffer,
-                      _(L"%ls: Not inside of 'if' block\n"),
-                      argv[0]);
-        builtin_print_help(parser, argv[0], stderr_buffer);
-        return STATUS_BUILTIN_ERROR;
-    }
-    else
-    {
-        /* Run the else block if the IF expression was false and so were all the ELSEIF expressions (if any) */
-        bool run_else = ! if_block->any_branch_taken;
-        if_block->skip = ! run_else;
-        if_block->else_evaluated = true;
-        env_pop();
-        env_push(false);
-    }
-
-    /*
-      If everything goes ok, return status of last command to execute.
-    */
-    return proc_get_last_status();
-}
-
 /**
    This function handles both the 'continue' and the 'break' builtins
    that are used for loop control.
@@ -4100,97 +3481,12 @@ static int builtin_return(parser_t &parser, wchar_t **argv)
     for (size_t i=0; i < function_block_idx; i++)
     {
         block_t *b = parser.block_at_index(i);
-        b->mark_as_fake();
         b->skip = true;
     }
     parser.block_at_index(function_block_idx)->skip = true;
     return status;
 }
 
-/**
-   Builtin for executing one of several blocks of commands depending
-   on the value of an argument.
-*/
-static int builtin_switch(parser_t &parser, wchar_t **argv)
-{
-    int res=STATUS_BUILTIN_OK;
-    int argc = builtin_count_args(argv);
-
-    /* Hackish - if we have no arguments other than the command, we are a "naked invocation" and we just print help */
-    if (argc == 1)
-    {
-        builtin_print_help(parser, argv[0], stdout_buffer);
-        return STATUS_BUILTIN_ERROR;
-    }
-
-    if (argc != 2)
-    {
-        append_format(stderr_buffer,
-                      _(L"%ls: Expected exactly one argument, got %d\n"),
-                      argv[0],
-                      argc-1);
-
-        builtin_print_help(parser, argv[0], stderr_buffer);
-        res=1;
-        parser.push_block(new fake_block_t());
-    }
-    else
-    {
-        parser.push_block(new switch_block_t(argv[1]));
-        parser.current_block()->skip=1;
-        res = proc_get_last_status();
-    }
-
-    return res;
-}
-
-/**
-   Builtin used together with the switch builtin for conditional
-   execution
-*/
-static int builtin_case(parser_t &parser, wchar_t **argv)
-{
-    int argc = builtin_count_args(argv);
-    int i;
-    wchar_t *unescaped=0;
-
-    if (parser.current_block()->type() != SWITCH)
-    {
-        append_format(stderr_buffer,
-                      _(L"%ls: 'case' command while not in switch block\n"),
-                      argv[0]);
-        builtin_print_help(parser, argv[0], stderr_buffer);
-        return STATUS_BUILTIN_ERROR;
-    }
-
-    parser.current_block()->skip = 1;
-    switch_block_t *sb = static_cast<switch_block_t *>(parser.current_block());
-    if (sb->switch_taken)
-    {
-        return proc_get_last_status();
-    }
-
-    const wcstring &switch_value = sb->switch_value;
-    for (i=1; i<argc; i++)
-    {
-        int match;
-
-        unescaped = parse_util_unescape_wildcards(argv[i]);
-        match = wildcard_match(switch_value, unescaped);
-        free(unescaped);
-
-        if (match)
-        {
-            parser.current_block()->skip = 0;
-            sb->switch_taken = true;
-            break;
-        }
-    }
-
-    return proc_get_last_status();
-}
-
-
 /**
    History of commands executed by user
 */
@@ -4387,14 +3683,14 @@ static const builtin_data_t builtin_datas[]=
     { 		L"[",  &builtin_test, N_(L"Test a condition")   },
     { 		L"__fish_parse",  &builtin_parse, N_(L"Try out the new parser")  },
     { 		L"and",  &builtin_generic, N_(L"Execute command if previous command suceeded")  },
-    { 		L"begin",  &builtin_begin, N_(L"Create a block of code")   },
+    { 		L"begin",  &builtin_generic, N_(L"Create a block of code")   },
     { 		L"bg",  &builtin_bg, N_(L"Send job to background")   },
     { 		L"bind",  &builtin_bind, N_(L"Handle fish key bindings")  },
     { 		L"block",  &builtin_block, N_(L"Temporarily block delivery of events") },
     { 		L"break",  &builtin_break_continue, N_(L"Stop the innermost loop")   },
     { 		L"breakpoint",  &builtin_breakpoint, N_(L"Temporarily halt execution of a script and launch an interactive debug prompt")   },
     { 		L"builtin",  &builtin_builtin, N_(L"Run a builtin command instead of a function") },
-    { 		L"case",  &builtin_case, N_(L"Conditionally execute a block of commands")   },
+    { 		L"case",  &builtin_generic, N_(L"Conditionally execute a block of commands")   },
     { 		L"cd",  &builtin_cd, N_(L"Change working directory")   },
     { 		L"command",   &builtin_generic, N_(L"Run a program instead of a function or builtin")   },
     { 		L"commandline",  &builtin_commandline, N_(L"Set or get the commandline")   },
@@ -4403,14 +3699,14 @@ static const builtin_data_t builtin_datas[]=
     { 		L"continue",  &builtin_break_continue, N_(L"Skip the rest of the current lap of the innermost loop")   },
     { 		L"count",  &builtin_count, N_(L"Count the number of arguments")   },
     {       L"echo",  &builtin_echo, N_(L"Print arguments") },
-    { 		L"else",  &builtin_else, N_(L"Evaluate block if condition is false")   },
+    { 		L"else",  &builtin_generic, N_(L"Evaluate block if condition is false")   },
     { 		L"emit",  &builtin_emit, N_(L"Emit an event") },
-    { 		L"end",  &builtin_end, N_(L"End a block of commands")   },
+    { 		L"end",  &builtin_generic, N_(L"End a block of commands")   },
     { 		L"exec",  &builtin_generic, N_(L"Run command in current process")  },
     { 		L"exit",  &builtin_exit, N_(L"Exit the shell") },
     { 		L"fg",  &builtin_fg, N_(L"Send job to foreground")   },
-    { 		L"for",  &builtin_for, N_(L"Perform a set of commands multiple times")   },
+    { 		L"for",  &builtin_generic, N_(L"Perform a set of commands multiple times")   },
-    { 		L"function",  &builtin_function, N_(L"Define a new function")   },
+    { 		L"function",  &builtin_generic, N_(L"Define a new function")   },
     { 		L"functions",  &builtin_functions, N_(L"List or remove functions")   },
     { 		L"history",  &builtin_history, N_(L"History of commands executed by user")   },
     { 		L"if",  &builtin_generic, N_(L"Evaluate block if condition is true")   },
@@ -4426,7 +3722,7 @@ static const builtin_data_t builtin_datas[]=
     { 		L"set_color",  &builtin_set_color, N_(L"Set the terminal color")   },
     { 		L"source",  &builtin_source, N_(L"Evaluate contents of file")   },
     { 		L"status",  &builtin_status, N_(L"Return status information about fish")  },
-    { 		L"switch",  &builtin_switch, N_(L"Conditionally execute a block of commands")   },
+    { 		L"switch",  &builtin_generic, N_(L"Conditionally execute a block of commands")   },
     { 		L"test",  &builtin_test, N_(L"Test a condition")   },
     { 		L"ulimit",  &builtin_ulimit, N_(L"Set or get the shells resource usage limits")  },
     { 		L"while",  &builtin_generic, N_(L"Perform a command multiple times")   }
@@ -4492,7 +3788,7 @@ int builtin_run(parser_t &parser, const wchar_t * const *argv, const io_chain_t
 
     if (argv[1] != 0 && !internal_help(argv[0]))
     {
-        if (argv[2] == 0 && (parser.is_help(argv[1], 0)))
+        if (argv[2] == 0 && (parse_util_argument_is_help(argv[1], 0)))
         {
             builtin_print_help(parser, argv[0], stdout_buffer);
             return STATUS_BUILTIN_OK;
@@ -4509,7 +3805,7 @@ int builtin_run(parser_t &parser, const wchar_t * const *argv, const io_chain_t
     }
     else
     {
-        debug(0, _(L"Unknown builtin '%ls'"), argv[0]);
+        debug(0, UNKNOWN_BUILTIN_ERR_MSG, argv[0]);
     }
     return STATUS_BUILTIN_ERROR;
 }

builtin.h

@@ -179,7 +179,7 @@ const wchar_t *builtin_complete_get_temporary_buffer();
 wcstring builtin_help_get(parser_t &parser, const wchar_t *cmd);
 
 /** Defines a function, like builtin_function. Returns 0 on success. args should NOT contain 'function' as the first argument. */
-int define_function(parser_t &parser, const wcstring_list_t &args, const wcstring &contents, wcstring *out_err);
+int define_function(parser_t &parser, const wcstring_list_t &args, const wcstring &contents, int definition_line_offset, wcstring *out_err);
 
 
 #endif

builtin_commandline.cpp

@@ -214,7 +214,8 @@ static int builtin_commandline(parser_t &parser, wchar_t **argv)
     int cursor_mode = 0;
     int line_mode = 0;
     int search_mode = 0;
-    const wchar_t *begin, *end;
+    int paging_mode = 0;
+    const wchar_t *begin = NULL, *end = NULL;
 
     current_buffer = (wchar_t *)builtin_complete_get_temporary_buffer();
     if (current_buffer)
@@ -252,75 +253,25 @@ static int builtin_commandline(parser_t &parser, wchar_t **argv)
         static const struct woption
                 long_options[] =
         {
-            {
+            { L"append", no_argument, 0, 'a' },
-                L"append", no_argument, 0, 'a'
+            { L"insert", no_argument, 0, 'i' },
-            }
+            { L"replace", no_argument, 0, 'r' },
-            ,
+            { L"current-job", no_argument, 0, 'j' },
-            {
+            { L"current-process", no_argument, 0, 'p' },
-                L"insert", no_argument, 0, 'i'
+            { L"current-token", no_argument, 0, 't' },
-            }
+            { L"current-buffer", no_argument, 0, 'b' },
-            ,
+            { L"cut-at-cursor", no_argument, 0, 'c' },
-            {
+            { L"function", no_argument, 0, 'f' },
-                L"replace", no_argument, 0, 'r'
+            { L"tokenize", no_argument, 0, 'o' },
-            }
+            { L"help", no_argument, 0, 'h' },
-            ,
+            { L"input", required_argument, 0, 'I' },
-            {
+            { L"cursor", no_argument, 0, 'C' },
-                L"current-job", no_argument, 0, 'j'
+            { L"line", no_argument, 0, 'L' },
-            }
+            { L"search-mode", no_argument, 0, 'S' },
-            ,
+            { L"selection", no_argument, 0, 's' },
-            {
+            { L"paging-mode", no_argument, 0, 'P' },
-                L"current-process", no_argument, 0, 'p'
+            { 0, 0, 0, 0 }
-            }
+        };
-            ,
-            {
-                L"current-token", no_argument, 0, 't'
-            }
-            ,
-            {
-                L"current-buffer", no_argument, 0, 'b'
-            }
-            ,
-            {
-                L"cut-at-cursor", no_argument, 0, 'c'
-            }
-            ,
-            {
-                L"function", no_argument, 0, 'f'
-            }
-            ,
-            {
-                L"tokenize", no_argument, 0, 'o'
-            }
-            ,
-            {
-                L"help", no_argument, 0, 'h'
-            }
-            ,
-            {
-                L"input", required_argument, 0, 'I'
-            }
-            ,
-            {
-                L"cursor", no_argument, 0, 'C'
-            }
-            ,
-            {
-                L"line", no_argument, 0, 'L'
-            }
-            ,
-            {
-                L"search-mode", no_argument, 0, 'S'
-            }
-            ,
-            {
-                L"selection", no_argument, 0, 's'
-            }
-            ,
-            {
-                0, 0, 0, 0
-            }
-        }
-        ;
 
         int opt_index = 0;
 
@@ -407,6 +358,10 @@ static int builtin_commandline(parser_t &parser, wchar_t **argv)
                 selection_mode = 1;
                 break;
                 
+            case 'P':
+                paging_mode = 1;
+                break;
+
             case 'h':
                 builtin_print_help(parser, argv[0], stdout_buffer);
                 return 0;
@@ -424,7 +379,7 @@ static int builtin_commandline(parser_t &parser, wchar_t **argv)
         /*
           Check for invalid switch combinations
         */
-        if (buffer_part || cut_at_cursor || append_mode || tokenize || cursor_mode || line_mode || search_mode)
+        if (buffer_part || cut_at_cursor || append_mode || tokenize || cursor_mode || line_mode || search_mode || paging_mode)
         {
             append_format(stderr_buffer,
                           BUILTIN_ERR_COMBO,
@@ -474,7 +429,7 @@ static int builtin_commandline(parser_t &parser, wchar_t **argv)
     {
         size_t start, len;
         const wchar_t *buffer = reader_get_buffer();
-        if(reader_get_selection(start, len))
+        if (reader_get_selection(&start, &len))
         {
           wchar_t *selection = new wchar_t[len + 1];
           selection[len] = L'\0';
@@ -493,7 +448,7 @@ static int builtin_commandline(parser_t &parser, wchar_t **argv)
     /*
       Check for invalid switch combinations
     */
-    if ((search_mode || line_mode || cursor_mode) && (argc-woptind > 1))
+    if ((search_mode || line_mode || cursor_mode || paging_mode) && (argc-woptind > 1))
     {
 
         append_format(stderr_buffer,
@@ -504,7 +459,7 @@ static int builtin_commandline(parser_t &parser, wchar_t **argv)
         return 1;
     }
 
-    if ((buffer_part || tokenize || cut_at_cursor) && (cursor_mode || line_mode || search_mode))
+    if ((buffer_part || tokenize || cut_at_cursor) && (cursor_mode || line_mode || search_mode || paging_mode))
     {
         append_format(stderr_buffer,
                       BUILTIN_ERR_COMBO,
@@ -596,6 +551,11 @@ static int builtin_commandline(parser_t &parser, wchar_t **argv)
         return !reader_search_mode();
     }
 
+    if (paging_mode)
+    {
+        return ! reader_has_pager_contents();
+    }
+
 
     switch (buffer_part)
     {

builtin_complete.cpp

@@ -312,75 +312,24 @@ static int builtin_complete(parser_t &parser, wchar_t **argv)
         static const struct woption
                 long_options[] =
         {
-            {
+            { L"exclusive", no_argument, 0, 'x' },
-                L"exclusive", no_argument, 0, 'x'
+            { L"no-files", no_argument, 0, 'f' },
-            }
+            { L"require-parameter", no_argument, 0, 'r' },
-            ,
+            { L"path", required_argument, 0, 'p' },
-            {
+            { L"command", required_argument, 0, 'c' },
-                L"no-files", no_argument, 0, 'f'
+            { L"short-option", required_argument, 0, 's' },
-            }
+            { L"long-option", required_argument, 0, 'l' },
-            ,
+            { L"old-option", required_argument, 0, 'o' },
-            {
+            { L"description", required_argument, 0, 'd' },
-                L"require-parameter", no_argument, 0, 'r'
+            { L"arguments", required_argument, 0, 'a' },
-            }
+            { L"erase", no_argument, 0, 'e' },
-            ,
+            { L"unauthoritative", no_argument, 0, 'u' },
-            {
+            { L"authoritative", no_argument, 0, 'A' },
-                L"path", required_argument, 0, 'p'
+            { L"condition", required_argument, 0, 'n' },
-            }
+            { L"do-complete", optional_argument, 0, 'C' },
-            ,
+            { L"help", no_argument, 0, 'h' },
-            {
+            { 0, 0, 0, 0 }
-                L"command", required_argument, 0, 'c'
+        };
-            }
-            ,
-            {
-                L"short-option", required_argument, 0, 's'
-            }
-            ,
-            {
-                L"long-option", required_argument, 0, 'l'
-            }
-            ,
-            {
-                L"old-option", required_argument, 0, 'o'
-            }
-            ,
-            {
-                L"description", required_argument, 0, 'd'
-            }
-            ,
-            {
-                L"arguments", required_argument, 0, 'a'
-            }
-            ,
-            {
-                L"erase", no_argument, 0, 'e'
-            }
-            ,
-            {
-                L"unauthoritative", no_argument, 0, 'u'
-            }
-            ,
-            {
-                L"authoritative", no_argument, 0, 'A'
-            }
-            ,
-            {
-                L"condition", required_argument, 0, 'n'
-            }
-            ,
-            {
-                L"do-complete", optional_argument, 0, 'C'
-            }
-            ,
-            {
-                L"help", no_argument, 0, 'h'
-            }
-            ,
-            {
-                0, 0, 0, 0
-            }
-        }
-        ;
 
         int opt_index = 0;
 
@@ -518,15 +467,22 @@ static int builtin_complete(parser_t &parser, wchar_t **argv)
     {
         if (comp && wcslen(comp))
         {
-            if (parser.test_args(comp, 0, 0))
+            wcstring prefix;
+            if (argv[0])
+            {
+                prefix.append(argv[0]);
+                prefix.append(L": ");
+            }
+
+            wcstring err_text;
+            if (parser.detect_errors_in_argument_list(comp, &err_text, prefix.c_str()))
             {
                 append_format(stderr_buffer,
                               L"%ls: Completion '%ls' contained a syntax error\n",
                               argv[0],
                               comp);
-
+                stderr_buffer.append(err_text);
-                parser.test_args(comp, &stderr_buffer, argv[0]);
+                stderr_buffer.push_back(L'\n');
-
                 res = true;
             }
         }
@@ -554,26 +510,28 @@ static int builtin_complete(parser_t &parser, wchar_t **argv)
                 {
                     const completion_t &next =  comp.at(i);
                     
-                    const wchar_t *prepend;
+                    /* Make a fake commandline, and then apply the completion to it.  */
+                    const wcstring faux_cmdline = token;
+                    size_t tmp_cursor = faux_cmdline.size();
+                    wcstring faux_cmdline_with_completion = completion_apply_to_command_line(next.completion, next.flags, faux_cmdline, &tmp_cursor, false);
                     
-                    if (next.flags & COMPLETE_REPLACES_TOKEN)
+                    /* completion_apply_to_command_line will append a space unless COMPLETE_NO_SPACE is set. We don't want to set COMPLETE_NO_SPACE because that won't close quotes. What we want is to close the quote, but not append the space. So we just look for the space and clear it. */
+                    if (! (next.flags & COMPLETE_NO_SPACE) && string_suffixes_string(L" ", faux_cmdline_with_completion))
                     {
-                        prepend = L"";
+                        faux_cmdline_with_completion.resize(faux_cmdline_with_completion.size() - 1);
-                    }
-                    else
-                    {
-                        prepend = token;
                     }
                     
+                    /* The input data is meant to be something like you would have on the command line, e.g. includes backslashes. The output should be raw, i.e. unescaped. So we need to unescape the command line. See #1127 */
+                    unescape_string_in_place(&faux_cmdline_with_completion, UNESCAPE_DEFAULT);
+                    stdout_buffer.append(faux_cmdline_with_completion);
                     
-                    if (!(next.description).empty())
+                    /* Append any description */
+                    if (! next.description.empty())
                     {
-                        append_format(stdout_buffer, L"%ls%ls\t%ls\n", prepend, next.completion.c_str(), next.description.c_str());
+                        stdout_buffer.push_back(L'\t');
-                    }
+                        stdout_buffer.append(next.description);
-                    else
-                    {
-                        append_format(stdout_buffer, L"%ls%ls\n", prepend, next.completion.c_str());
                     }
+                    stdout_buffer.push_back(L'\n');
                 }
 
                 recursion_level--;

builtin_set_color.cpp

@@ -235,9 +235,8 @@ static int builtin_set_color(parser_t &parser, wchar_t **argv)
     output_set_writer(saved_writer_func);
 
     /* Output the collected string */
-    std::string local_output;
+    stdout_buffer.append(str2wcstring(builtin_set_color_output));
-    std::swap(builtin_set_color_output, local_output);
+    builtin_set_color_output.clear();
-    stdout_buffer.append(str2wcstring(local_output));
 
     return STATUS_BUILTIN_OK;
 }

common.cpp

@@ -83,7 +83,7 @@ static bool thread_assertions_configured_for_testing = false;
 wchar_t ellipsis_char;
 wchar_t omitted_newline_char;
 
-char *profile=0;
+bool g_profiling_active = false;
 
 const wchar_t *program_name;
 
@@ -576,7 +576,7 @@ wcstring wsetlocale(int category, const wchar_t *locale)
         return format_string(L"%s", res);
 }
 
-bool contains_internal(const wchar_t *a, ...)
+bool contains_internal(const wchar_t *a, int vararg_handle, ...)
 {
     const wchar_t *arg;
     va_list va;
@@ -584,7 +584,7 @@ bool contains_internal(const wchar_t *a, ...)
 
     CHECK(a, 0);
 
-    va_start(va, a);
+    va_start(va, vararg_handle);
     while ((arg=va_arg(va, const wchar_t *))!= 0)
     {
         if (wcscmp(a,arg) == 0)
@@ -598,17 +598,19 @@ bool contains_internal(const wchar_t *a, ...)
     return res;
 }
 
-/* wcstring variant of contains_internal. The first parameter is a wcstring, the rest are const wchar_t* */
+/* wcstring variant of contains_internal. The first parameter is a wcstring, the rest are const wchar_t *. vararg_handle exists only to give us a POD-value to apss to va_start */
-__sentinel bool contains_internal(const wcstring &needle, ...)
+__sentinel bool contains_internal(const wcstring &needle, int vararg_handle, ...)
 {
     const wchar_t *arg;
     va_list va;
     int res = 0;
 
-    va_start(va, needle);
+    const wchar_t *needle_cstr = needle.c_str();
+    va_start(va, vararg_handle);
     while ((arg=va_arg(va, const wchar_t *))!= 0)
     {
-        if (needle == arg)
+        /* libc++ has an unfortunate implementation of operator== that unconditonally wcslen's the wchar_t* parameter, so prefer wcscmp directly */
+        if (! wcscmp(needle_cstr, arg))
         {
             res=1;
             break;
@@ -711,6 +713,11 @@ void debug(int level, const char *msg, ...)
     errno = errno_old;
 }
 
+void print_stderr(const wcstring &str)
+{
+    fprintf(stderr, "%ls\n", str.c_str());
+}
+
 
 void debug_safe(int level, const char *msg, const char *param1, const char *param2, const char *param3, const char *param4, const char *param5, const char *param6, const char *param7, const char *param8, const char *param9, const char *param10, const char *param11, const char *param12)
 {
@@ -1674,14 +1681,17 @@ int common_get_height()
 void tokenize_variable_array(const wcstring &val, std::vector<wcstring> &out)
 {
     size_t pos = 0, end = val.size();
-    while (pos < end)
+    while (pos <= end)
     {
         size_t next_pos = val.find(ARRAY_SEP, pos);
-        if (next_pos == wcstring::npos) break;
+        if (next_pos == wcstring::npos)
-        out.push_back(val.substr(pos, next_pos - pos));
+        {
-        pos = next_pos + 1; //skip the separator
+            next_pos = end;
+        }
+        out.resize(out.size() + 1);
+        out.back().assign(val, pos, next_pos - pos);
+        pos = next_pos + 1; //skip the separator, or skip past the end
     }
-    out.push_back(val.substr(pos, end - pos));
 }
 
 bool string_prefixes_string(const wchar_t *proposed_prefix, const wcstring &value)

common.h

@@ -87,6 +87,37 @@ enum
 };
 typedef unsigned int escape_flags_t;
 
+/* Directions */
+enum selection_direction_t
+{
+    /* visual directions */
+    direction_north,
+    direction_east,
+    direction_south,
+    direction_west,
+    
+    /* logical directions */
+    direction_next,
+    direction_prev,
+    
+    /* special value that means deselect */
+    direction_deselect
+};
+
+inline bool selection_direction_is_cardinal(selection_direction_t dir)
+{
+    switch (dir)
+    {
+        case direction_north:
+        case direction_east:
+        case direction_south:
+        case direction_west:
+            return true;
+        default:
+            return false;
+    }
+}
+
 /**
  Helper macro for errors
  */
@@ -118,7 +149,7 @@ extern int debug_level;
 /**
    Profiling flag. True if commands should be profiled.
 */
-extern char *profile;
+extern bool g_profiling_active;
 
 /**
    Name of the current program. Should be set at startup. Used by the
@@ -197,7 +228,7 @@ extern const wchar_t *program_name;
 /**
    Check if the specified string element is a part of the specified string list
  */
-#define contains( str,... ) contains_internal( str, __VA_ARGS__, NULL )
+#define contains( str, ... ) contains_internal( str, 0, __VA_ARGS__, NULL )
 
 /**
   Print a stack trace to stderr
@@ -657,8 +688,8 @@ wcstring wsetlocale(int category, const wchar_t *locale);
 
    \return zero if needle is not found, of if needle is null, non-zero otherwise
 */
-__sentinel bool contains_internal(const wchar_t *needle, ...);
+__sentinel bool contains_internal(const wchar_t *needle, int vararg_handle, ...);
-__sentinel bool contains_internal(const wcstring &needle, ...);
+__sentinel bool contains_internal(const wcstring &needle, int vararg_handle, ...);
 
 /**
    Call read while blocking the SIGCHLD signal. Should only be called
@@ -701,6 +732,9 @@ ssize_t read_loop(int fd, void *buff, size_t count);
 void debug(int level, const char *msg, ...);
 void debug(int level, const wchar_t *msg, ...);
 
+/** Writes a string to stderr, followed by a newline */
+void print_stderr(const wcstring &str);
+
 /**
    Replace special characters with backslash escape sequences. Newline is
    replaced with \n, etc.

complete.cpp

@@ -967,7 +967,7 @@ void completer_t::complete_strings(const wcstring &wc_escaped,
                                    complete_flags_t flags)
 {
     wcstring tmp = wc_escaped;
-    if (! expand_one(tmp, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_WILDCARDS | this->expand_flags()))
+    if (! expand_one(tmp, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_WILDCARDS | this->expand_flags(), NULL))
         return;
 
     const wchar_t *wc = parse_util_unescape_wildcards(tmp.c_str());
@@ -1146,7 +1146,7 @@ void completer_t::complete_cmd(const wcstring &str_cmd, bool use_function, bool
     if (use_command)
     {
 
-        if (expand_string(str_cmd, this->completions, ACCEPT_INCOMPLETE | EXECUTABLES_ONLY | this->expand_flags()) != EXPAND_ERROR)
+        if (expand_string(str_cmd, this->completions, ACCEPT_INCOMPLETE | EXECUTABLES_ONLY | this->expand_flags(), NULL) != EXPAND_ERROR)
         {
             if (this->wants_descriptions())
             {
@@ -1179,7 +1179,7 @@ void completer_t::complete_cmd(const wcstring &str_cmd, bool use_function, bool
                     size_t prev_count =  this->completions.size();
                     if (expand_string(nxt_completion,
                                       this->completions,
-                                      ACCEPT_INCOMPLETE | EXECUTABLES_ONLY | this->expand_flags()) != EXPAND_ERROR)
+                                      ACCEPT_INCOMPLETE | EXECUTABLES_ONLY | this->expand_flags(), NULL) != EXPAND_ERROR)
                     {
                         /* For all new completions, if COMPLETE_NO_CASE is set, then use only the last path component */
                         for (size_t i=prev_count; i< this->completions.size(); i++)
@@ -1249,7 +1249,7 @@ void completer_t::complete_from_args(const wcstring &str,
     if (! is_autosuggest)
         proc_push_interactive(0);
 
-    parser.eval_args(args.c_str(), possible_comp);
+    parser.expand_argument_list(args, possible_comp);
 
     if (! is_autosuggest)
         proc_pop_interactive();
@@ -1641,7 +1641,7 @@ void completer_t::complete_param_expand(const wcstring &sstr, bool do_file)
 
     if (expand_string(comp_str,
                       this->completions,
-                      flags) == EXPAND_ERROR)
+                      flags, NULL) == EXPAND_ERROR)
     {
         debug(3, L"Error while expanding string '%ls'", comp_str);
     }
@@ -1712,29 +1712,67 @@ bool completer_t::complete_variable(const wcstring &str, size_t start_offset)
     return res;
 }
 
-/**
-   Search the specified string for the \$ sign. If found, try to
-   complete as an environment variable.
-
-   \return 0 if unable to complete, 1 otherwise
-*/
 bool completer_t::try_complete_variable(const wcstring &str)
 {
-    size_t i = str.size();
+    enum {e_unquoted, e_single_quoted, e_double_quoted} mode = e_unquoted;
-    while (i--)
+    const size_t len = str.size();
+    
+    /* Get the position of the dollar heading a run of valid variable characters. -1 means none. */
+    size_t variable_start = -1;
+    
+    for (size_t in_pos=0; in_pos<len; in_pos++)
     {
-        wchar_t c = str.at(i);
+        wchar_t c = str.at(in_pos);
-        if (c == L'$')
+        if (! wcsvarchr(c))
         {
-            /*      wprintf( L"Var prefix \'%ls\'\n", &cmd[i+1] );*/
+            /* This character cannot be in a variable, reset the dollar */
-            return this->complete_variable(str, i+1);
+            variable_start = -1;
         }
-        if (!isalnum(c) && c != L'_')
+        
+        switch (c)
         {
-            return false;
+            case L'\\':
+                in_pos++;
+                break;
+            
+            case L'$':
+                if (mode == e_unquoted || mode == e_double_quoted)
+                {
+                    variable_start = in_pos;
+                }
+                break;
+            
+            case L'\'':
+                if (mode == e_single_quoted)
+                {
+                    mode = e_unquoted;
+                }
+                else if (mode == e_unquoted)
+                {
+                    mode = e_single_quoted;
+                }
+                break;
+            
+            case L'"':
+                if (mode == e_double_quoted)
+                {
+                    mode = e_unquoted;
+                }
+                else if (mode == e_unquoted)
+                {
+                    mode = e_double_quoted;
+                }
+                break;
         }
     }
-    return false;
+    
+    /* Now complete if we have a variable start that's also not the last character */
+    bool result = false;
+    if (variable_start != static_cast<size_t>(-1) && variable_start + 1 < len)
+    {
+        result = this->complete_variable(str, variable_start + 1);
+    }
+    return result;
 }
 
 /**
@@ -1822,7 +1860,7 @@ void complete(const wcstring &cmd_with_subcmds, std::vector<completion_t> &comps
     bool use_function = 1;
     bool use_builtin = 1;
 
-    //  debug( 1, L"Complete '%ls'", cmd );
+      //debug( 1, L"Complete '%ls'", cmd.c_str() );
 
     const wchar_t *cmd_cstr = cmd.c_str();
     const wchar_t *tok_begin = NULL, *prev_begin = NULL, *prev_end = NULL;
@@ -1832,9 +1870,9 @@ void complete(const wcstring &cmd_with_subcmds, std::vector<completion_t> &comps
      If we are completing a variable name or a tilde expansion user
      name, we do that and return. No need for any other completions.
      */
-
     const wcstring current_token = tok_begin;
 
+    /* Unconditionally complete variables and processes. This is a little weird since we will happily complete variables even in e.g. command position, despite the fact that they are invalid there. */
     if (!done)
     {
         done = completer.try_complete_variable(current_token) || completer.try_complete_user(current_token);
@@ -1848,9 +1886,21 @@ void complete(const wcstring &cmd_with_subcmds, std::vector<completion_t> &comps
         parse_node_tree_t tree;
         parse_tree_from_string(cmd, parse_flag_continue_after_error | parse_flag_accept_incomplete_tokens, &tree, NULL);
         
-        /* Find the plain statement that contains the position */
+        /* Find any plain statement that contains the position. We have to backtrack past spaces (#1261). So this will be at either the last space character, or after the end of the string */
-        const parse_node_t *plain_statement = tree.find_node_matching_source_location(symbol_plain_statement, pos, NULL);
+        size_t adjusted_pos = pos;
-        if (plain_statement != NULL)
+        while (adjusted_pos > 0 && cmd.at(adjusted_pos - 1) == L' ')
+        {
+            adjusted_pos--;
+        }
+        
+        const parse_node_t *plain_statement = tree.find_node_matching_source_location(symbol_plain_statement, adjusted_pos, NULL);
+        if (plain_statement == NULL)
+        {
+            /* Not part of a plain statement. This could be e.g. a for loop header, case expression, etc. Do generic file completions (#1309). If we had to backtrack, it means there was whitespace; don't do an autosuggestion in that case. */
+            bool no_file = (flags & COMPLETION_REQUEST_AUTOSUGGESTION) && (adjusted_pos < pos);
+            completer.complete_param_expand(current_token, ! no_file);
+        }
+        else
         {
             assert(plain_statement->has_source() && plain_statement->type == symbol_plain_statement);
 
@@ -1871,6 +1921,7 @@ void complete(const wcstring &cmd_with_subcmds, std::vector<completion_t> &comps
                     break;
 
                 case parse_statement_decoration_command:
+                case parse_statement_decoration_exec:
                     use_command = true;
                     use_function = false;
                     use_builtin = false;
@@ -1945,7 +1996,9 @@ void complete(const wcstring &cmd_with_subcmds, std::vector<completion_t> &comps
 
                 /* And if we're autosuggesting, and the token is empty, don't do file suggestions */
                 if ((flags & COMPLETION_REQUEST_AUTOSUGGESTION) && current_argument_unescape.empty())
+                {
                     do_file = false;
+                }
 
                 /* This function wants the unescaped string */
                 completer.complete_param_expand(current_token, do_file);

complete.h

@@ -14,6 +14,7 @@
 
 
 #include <wchar.h>
+#include <stdint.h>
 
 #include "util.h"
 #include "common.h"
@@ -54,13 +55,6 @@
  */
 #define COMPLETE_SEP_STR L"\004"
 
-/**
- * Separator between completion items in fish_pager. This is used for
- * completion grouping, e.g. when putting completions with the same
- * descriptions on the same line.
- */
-#define COMPLETE_ITEM_SEP L'\uf500'
-
 /**
  * Character that separates the completion and description on
  * programmable completions

configure.ac

@@ -26,7 +26,6 @@ AC_SUBST(HAVE_DOXYGEN)
 AC_SUBST(LDFLAGS_FISH)
 AC_SUBST(LIBS_FISH)
 AC_SUBST(LIBS_FISH_INDENT)
-AC_SUBST(LIBS_FISH_PAGER)
 AC_SUBST(LIBS_FISHD)
 AC_SUBST(LIBS_MIMEDB)
 
@@ -106,8 +105,6 @@ echo "CXXFLAGS: $CXXFLAGS"
 #
 # This mostly helps OS X users, since fink usually installs out of
 # tree and doesn't update CXXFLAGS.
-#
-# It also helps FreeBSD which puts libiconv in /usr/local/lib
 
 for i in /usr/pkg /sw /opt /opt/local /usr/local; do
 
@@ -152,9 +149,11 @@ AC_CONFIG_HEADERS(config.h)
 AH_BOTTOM([#if __GNUC__ >= 3
 #define __warn_unused __attribute__ ((warn_unused_result))
 #define __sentinel __attribute__ ((sentinel))
+#define __packed __attribute__ ((packed))
 #else
 #define __warn_unused
 #define __sentinel
+#define __packed
 #endif])
 
 
@@ -408,7 +407,7 @@ AC_DEFINE(
 AC_SEARCH_LIBS( connect, socket, , [AC_MSG_ERROR([Cannot find the socket library, needed to build this package.] )] )
 AC_SEARCH_LIBS( nanosleep, rt, , [AC_MSG_ERROR([Cannot find the rt library, needed to build this package.] )] )
 AC_SEARCH_LIBS( pthread_create, pthread, , [AC_MSG_ERROR([Cannot find the pthread library, needed to build this package.] )] )
-AC_SEARCH_LIBS( setupterm, [ncurses curses], , [AC_MSG_ERROR([Could not find a curses implementation, needed to build fish. If this is Linux, try running 'sudo apt-get install libncurses5-dev' or 'sudo yum install ncurses-devel'])] )
+AC_SEARCH_LIBS( setupterm, [ncurses tinfo curses], , [AC_MSG_ERROR([Could not find a curses implementation, needed to build fish. If this is Linux, try running 'sudo apt-get install libncurses5-dev' or 'sudo yum install ncurses-devel'])] )
 AC_SEARCH_LIBS( [nan], [m], [AC_DEFINE( [HAVE_NAN], [1], [Define to 1 if you have the nan function])] )
 
 if test x$local_gettext != xno; then
@@ -422,10 +421,6 @@ LIBS_SHARED=$LIBS
 #
 
 LIBS="$LIBS_SHARED"
-# Check for libiconv_open if we can't find iconv_open. Silly OS X does
-# weird macro magic for the sole purpose of amusing me.
-AC_SEARCH_LIBS( iconv_open, iconv, , [AC_SEARCH_LIBS( libiconv_open, iconv, , [AC_MSG_ERROR([Could not find an iconv implementation, needed to build fish])] )] )
-
 LIBS_FISH=$LIBS
 
 #
@@ -435,20 +430,11 @@ LIBS_FISH=$LIBS
 LIBS="$LIBS_SHARED"
 LIBS_FISH_INDENT=$LIBS
 
-#
-# Check for libraries needed by fish_pager.
-#
-
-LIBS="$LIBS_SHARED"
-AC_SEARCH_LIBS( iconv_open, iconv, , [AC_SEARCH_LIBS( libiconv_open, iconv, , [AC_MSG_ERROR([Could not find an iconv implementation, needed to build fish])] )] )
-LIBS_FISH_PAGER=$LIBS
-
 #
 # Check for libraries needed by fishd.
 #
 
 LIBS="$LIBS_SHARED"
-AC_SEARCH_LIBS( iconv_open, iconv, , [AC_SEARCH_LIBS( libiconv_open, iconv, , [AC_MSG_ERROR([Could not find an iconv implementation, needed to build fish])] )] )
 LIBS_FISHD=$LIBS
 
 #

doc_src/commandline.txt

@@ -57,6 +57,16 @@ If \c commandline is called during a call to complete a given string
 using <code>complete -C STRING</code>, \c commandline will consider the
 specified string to be the current contents of the command line.
 
+The following options output metadata about the commandline state:
+
+- \c -L or \c --line print the line that the cursor is on, with the topmost
+line starting at 1
+- \c -S or \c --search-mode evaluates to true if the commandline is performing
+a history search
+- \c -P or \c --paging-mode evaluates to true if the commandline is showing
+pager contents, such as tab completions
+
+
 \subsection commandline-example Example
 
 <tt>commandline -j $history[3]</tt> replaces the job under the cursor with the

doc_src/faq.hdr

@@ -266,7 +266,7 @@ Next, do the following (assuming fish was installed to /usr/local):
 rm -Rf /usr/local/etc/fish /usr/local/share/fish ~/.config/fish
 rm /usr/local/share/man/man1/fish*.1
 cd /usr/local/bin
-rm -f fish mimedb fish_pager fishd fish_indent
+rm -f fish mimedb fishd fish_indent
 </pre>
 
 */

doc_src/fish_pager.txt

@@ -1,7 +0,0 @@
-\section fish_pager fish_pager - internal command used by fish
-
-\subsection fish_pager-description Description
-
-\c fish_pager is used internally by fish. It should not be used by other
-commands, as its interface is liable to change in the future.
-

doc_src/index.hdr.in

@@ -387,8 +387,9 @@ cursor, in a muted gray color (which can be changed with the
 <code>fish_color_autosuggestion</code> variable).
 
 To accept the autosuggestion (replacing the command line contents),
-hit right arrow or Control-F. If the autosuggestion is not what you want,
+press right arrow or Control-F. To accept the first suggested word, press
-just ignore it: it won't execute unless you accept it.
+Alt-Right or Alt-F. If the autosuggestion is not what you want, just ignore it:
+it won't execute unless you accept it.
 
 Autosuggestions are a powerful way to quickly summon frequently entered commands, by
 typing the first few characters. They are also an efficient technique for navigating
@@ -1123,7 +1124,7 @@ Here are some of the commands available in the editor:
 - Home or Ctrl-A moves the cursor to the beginning of the line.
 - End or Ctrl-E moves to the end of line. If the cursor is already at the end of the line, and an autosuggestion is available, End or Ctrl-E accepts the autosuggestion.
 - Left (or Ctrl-B) and Right (or Ctrl-F) move the cursor left or right by one character. If the cursor is already at the end of the line, and an autosuggestion is available, the Right key and the Ctrl-F combination accept the suggestion.
-- Alt-Left and Alt-Right move the cursor one word left or right, or moves forward/backward in the directory history if the command line is empty.
+- Alt-Left and Alt-Right move the cursor one word left or right, or moves forward/backward in the directory history if the command line is empty. If the cursor is already at the end of the line, and an autosuggestion is available, Alt-Right (or Alt-F) accepts the first word in the suggestion.
 - Up and Down search the command history for the previous/next command containing the string that was specified on the commandline before the search was started. If the commandline was empty when the search started, all commands match. See the <a href='#history'>history </a>section for more information on history searching.
 - Alt-Up and Alt-Down search the command history for the previous/next token containing the token under the cursor before the search was started. If the commandline was not on a token when the search started, all tokens match. See the <a href='#history'>history </a>section for more information on history searching.
 - Delete and Backspace removes one character forwards or backwards respectively.

doc_src/isatty.txt

@@ -1,14 +1,29 @@
-\section isatty isatty - test if the specified file descriptor is a tty
+\section isatty isatty - test if a file or file descriptor is a tty.
 
 \subsection isatty-synopsis Synopsis
- <tt>isatty [FILE DESCRIPTOR]</tt>
+<tt>isatty [FILE | DEVICE | FILE DESCRIPTOR NUMBER]</tt>
 
 \subsection isatty-description Description
-<tt>isatty</tt> tests if a file descriptor is a tty.
+<tt>isatty</tt> tests if a file or file descriptor is a tty.
+The argument may be in the form of a file path, device, or file descriptor
+number. Without an argument, <tt>standard input</tt> is implied.
 
-<tt>FILE DESCRIPTOR</tt> may be either the number of a file descriptor, or one of the
+If the resolved file descriptor is a tty, the command returns zero. Otherwise, the command exits one. No messages are printed to standard error.
-strings <tt>stdin</tt>, \c stdout and <tt>stderr</tt>.
 
-If the specified file descriptor is a tty, the exit status of the command is
+\subsection isatty-examples Examples
-zero. Otherwise, it is non-zero.
 
+From an interactive shell, the commands below exit with a return value of zero:
+<pre>
+isatty
+isatty stdout
+isatty 2
+echo | isatty /dev/fd/1
+</pre>
+
+And these will exit non-zero:
+<pre>
+echo | isatty
+isatty /dev/fd/9
+isatty stdout > file
+isatty 2 2> file
+</pre>

doc_src/license.hdr

@@ -1402,4 +1402,20 @@ POSSIBILITY OF SUCH DAMAGES.
 
 */
 
+<h2>License for UTF8</h2>
+
+<p>Copyright (c) 2007 Alexey Vatchenko <av@bsdua.org>
+
+<p>Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+<p>THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
 \htmlonly </div> \endhtmlonly

doc_src/test.txt

@@ -97,4 +97,11 @@ end.
 <a href="http://www.unix.com/man-page/POSIX/1/test/">IEEE Std 1003.1-2008
 (POSIX.1) standard</a>. The following exceptions apply:
 - The \c < and \c > operators for comparing strings are not implemented.
+- Because this test is a shell builtin and not a standalone utility, using
+ the -c flag on a special file descriptors like standard input and output
+ may not return the same result when invoked from within a pipe as one
+ would expect when invoking the \c test utility in another shell.
+ 
+ In cases such as this, one can use \c command \c test to explicitly
+ use the system's standalone \c test rather than this \c builtin \c test.
 

doc_src/tutorial.hdr

@@ -349,7 +349,7 @@ And history too. Type a command once, and you can re-summon it by just typing a
 > <b>r</b><span class="suggest"><u>s</u>ync -avze ssh . myname@somelonghost.com:/some/long/path/doo/dee/doo/dee/doo</span>
 </pre>
 
-To accept the autosuggestion, hit right arrow or Control-F. If the autosuggestion is not what you want, just ignore it.
+To accept the autosuggestion, hit right arrow or Control-F. To accept a single word of the autosuggestion, hit Alt+right arrow. If the autosuggestion is not what you want, just ignore it.
 
 <h2 id="tut_tab_completions">Tab Completions</h2>
 

env_universal_common.cpp

@@ -21,7 +21,6 @@
 #include <sys/stat.h>
 #include <dirent.h>
 #include <wctype.h>
-#include <iconv.h>
 
 #include <errno.h>
 #include <locale.h>
@@ -39,6 +38,7 @@
 
 #include "common.h"
 #include "wutil.h"
+#include "utf8.h"
 #include "env_universal_common.h"
 
 /**
@@ -116,303 +116,28 @@ static void (*callback)(fish_message_type_t type,
                         const wchar_t *key,
                         const wchar_t *val);
 
-/**
+/* UTF <-> wchar conversions. These return a string allocated with malloc. These call sites could be cleaned up substantially to eliminate the dependence on malloc. */
-   List of names for the UTF-8 character set.
+static wchar_t *utf2wcs(const char *input)
- */
-static const char *iconv_utf8_names[]=
 {
-    "utf-8", "UTF-8",
+    wchar_t *result = NULL;
-    "utf8", "UTF8",
+    wcstring converted;
-    0
+    if (utf8_to_wchar_string(input, &converted))
+    {
+        result = wcsdup(converted.c_str());
     }
-;
+    return result;
-
-/**
-    List of wide character names, undefined byte length.
- */
-static const char *iconv_wide_names_unknown[]=
-{
-    "wchar_t", "WCHAR_T",
-    "wchar", "WCHAR",
-    0
-}
-;
-
-/**
-   List of wide character names, 4 bytes long.
- */
-static const char *iconv_wide_names_4[]=
-{
-    "wchar_t", "WCHAR_T",
-    "wchar", "WCHAR",
-    "ucs-4", "UCS-4",
-    "ucs4", "UCS4",
-    "utf-32", "UTF-32",
-    "utf32", "UTF32",
-    0
-}
-;
-
-/**
-   List of wide character names, 2 bytes long.
- */
-static const char *iconv_wide_names_2[]=
-{
-    "wchar_t", "WCHAR_T",
-    "wchar", "WCHAR",
-    "ucs-2", "UCS-2",
-    "ucs2", "UCS2",
-    "utf-16", "UTF-16",
-    "utf16", "UTF16",
-    0
-}
-;
-
-template<class T>
-class sloppy {};
-
-static size_t hack_iconv(iconv_t cd, const char * const* inbuf, size_t *inbytesleft, char **outbuf, size_t *outbytesleft)
-{
-    /* FreeBSD has this prototype: size_t iconv (iconv_t, const char **...)
-       OS X and Linux this one: size_t iconv (iconv_t, char **...)
-       AFAIK there's no single type that can be passed as both char ** and const char **.
-       Therefore, we let C++ figure it out, by providing a struct with an implicit conversion to both char** and const char **.
-    */
-    struct sloppy_char
-    {
-        const char * const * t;
-        operator char** () const
-        {
-            return (char **)t;
-        }
-        operator const char** () const
-        {
-            return (const char**)t;
-        }
-    } slop_inbuf = {inbuf};
-
-    return iconv(cd, slop_inbuf, inbytesleft, outbuf, outbytesleft);
 }
 
-/**
+static char *wcs2utf(const wchar_t *input)
-   Convert utf-8 string to wide string
- */
-static wchar_t *utf2wcs(const char *in)
 {
-    iconv_t cd=(iconv_t) -1;
+    char *result = NULL;
-    int i,j;
+    std::string converted;
-
+    if (wchar_to_utf8_string(input, &converted))
-    wchar_t *out;
-
-    /*
-      Try to convert to wchar_t. If that is not a valid character set,
-      try various names for ucs-4. We can't be sure that ucs-4 is
-      really the character set used by wchar_t, but it is the best
-      assumption we can make.
-    */
-    const char **to_name=0;
-
-    switch (sizeof(wchar_t))
     {
-
+        result = strdup(converted.c_str());
-        case 2:
-            to_name = iconv_wide_names_2;
-            break;
-
-        case 4:
-            to_name = iconv_wide_names_4;
-            break;
-
-        default:
-            to_name = iconv_wide_names_unknown;
-            break;
     }
-
+    return result;
-
-    /*
-      The line protocol fish uses is always utf-8.
-    */
-    const char **from_name = iconv_utf8_names;
-
-    size_t in_len = strlen(in);
-    size_t out_len =  sizeof(wchar_t)*(in_len+2);
-    size_t nconv;
-    char *nout;
-
-    out = (wchar_t *)malloc(out_len);
-    nout = (char *)out;
-
-    if (!out)
-        return 0;
-
-    for (i=0; to_name[i]; i++)
-    {
-        for (j=0; from_name[j]; j++)
-        {
-            cd = iconv_open(to_name[i], from_name[j]);
-
-            if (cd != (iconv_t) -1)
-            {
-                goto start_conversion;
-
 }
-        }
-    }
-
-start_conversion:
-
-    if (cd == (iconv_t) -1)
-    {
-        /* Something went wrong.  */
-        debug(0, L"Could not perform utf-8 conversion");
-        if (errno != EINVAL)
-            wperror(L"iconv_open");
-
-        /* Terminate the output string.  */
-        free(out);
-        return 0;
-    }
-
-    /* FreeBSD has this prototype: size_t iconv (iconv_t, const char **...)
-       OS X and Linux this one: size_t iconv (iconv_t, char **...)
-       AFAIK there's no single type that can be passed as both char ** and const char **.
-       Hence this hack.
-    */
-    nconv = hack_iconv(cd, &in, &in_len, &nout, &out_len);
-
-    if (nconv == (size_t) -1)
-    {
-        debug(0, L"Error while converting from utf string");
-        return 0;
-    }
-
-    *((wchar_t *) nout) = L'\0';
-
-    /*
-      Check for silly iconv behaviour inserting an bytemark in the output
-      string.
-     */
-    if (*out == L'\xfeff' || *out == L'\xffef' || *out == L'\xefbbbf')
-    {
-        wchar_t *out_old = out;
-        out = wcsdup(out+1);
-        if (! out)
-        {
-            debug(0, L"FNORD!!!!");
-            free(out_old);
-            return 0;
-        }
-        free(out_old);
-    }
-
-
-    if (iconv_close(cd) != 0)
-        wperror(L"iconv_close");
-
-    return out;
-}
-
-
-
-/**
-   Convert wide string to utf-8
- */
-static char *wcs2utf(const wchar_t *in)
-{
-    iconv_t cd=(iconv_t) -1;
-    int i,j;
-
-    char *char_in = (char *)in;
-    char *out;
-
-    /*
-      Try to convert to wchar_t. If that is not a valid character set,
-      try various names for ucs-4. We can't be sure that ucs-4 is
-      really the character set used by wchar_t, but it is the best
-      assumption we can make.
-    */
-    const char **from_name=0;
-
-    switch (sizeof(wchar_t))
-    {
-
-        case 2:
-            from_name = iconv_wide_names_2;
-            break;
-
-        case 4:
-            from_name = iconv_wide_names_4;
-            break;
-
-        default:
-            from_name = iconv_wide_names_unknown;
-            break;
-    }
-
-    const char **to_name = iconv_utf8_names;
-
-    size_t in_len = wcslen(in);
-    size_t out_len =  sizeof(char)*((MAX_UTF8_BYTES*in_len)+1);
-    size_t nconv;
-    char *nout;
-
-    out = (char *)malloc(out_len);
-    nout = (char *)out;
-    in_len *= sizeof(wchar_t);
-
-    if (!out)
-        return 0;
-
-    for (i=0; to_name[i]; i++)
-    {
-        for (j=0; from_name[j]; j++)
-        {
-            cd = iconv_open(to_name[i], from_name[j]);
-
-            if (cd != (iconv_t) -1)
-            {
-                goto start_conversion;
-
-            }
-        }
-    }
-
-start_conversion:
-
-    if (cd == (iconv_t) -1)
-    {
-        /* Something went wrong.  */
-        debug(0, L"Could not perform utf-8 conversion");
-        if (errno != EINVAL)
-            wperror(L"iconv_open");
-
-        /* Terminate the output string.  */
-        free(out);
-        return 0;
-    }
-
-    nconv = hack_iconv(cd, &char_in, &in_len, &nout, &out_len);
-
-
-    if (nconv == (size_t) -1)
-    {
-        debug(0, L"%d %d", in_len, out_len);
-        debug(0, L"Error while converting from to string");
-
-        /* Terminate the output string.  */
-        free(out);
-        return 0;
-    }
-
-    *nout = '\0';
-
-    if (iconv_close(cd) != 0)
-        wperror(L"iconv_close");
-
-    return out;
-}
-
-
 
 void env_universal_common_init(void (*cb)(fish_message_type_t type, const wchar_t *key, const wchar_t *val))
 {
@@ -756,8 +481,9 @@ static wcstring full_escape(const wchar_t *in)
         {
             out.push_back(c);
         }
-        else if (c < 256)
+        else if (c <= ASCII_MAX)
         {
+            // See #1225 for discussion of use of ASCII_MAX here
             append_format(out, L"\\x%.2x", c);
         }
         else if (c < 65536)

event.cpp

@@ -58,7 +58,7 @@ signal_list_t;
   active, which is the one that new events is written to. The inactive
   one contains the events that are currently beeing performed.
 */
-static signal_list_t sig_list[]= {{0,0},{0,0}};
+static signal_list_t sig_list[]= {{},{}};
 
 /**
    The index of sig_list that is the list of signals currently written to

exec.cpp

@@ -929,7 +929,7 @@ void exec_job(parser_t &parser, job_t *j)
                 if (p->next)
                 {
                     // Be careful to handle failure, e.g. too many open fds
-                    block_output_io_buffer.reset(io_buffer_t::create(false /* = not input */, STDOUT_FILENO));
+                    block_output_io_buffer.reset(io_buffer_t::create(STDOUT_FILENO));
                     if (block_output_io_buffer.get() == NULL)
                     {
                         exec_error = true;
@@ -958,7 +958,7 @@ void exec_job(parser_t &parser, job_t *j)
             {
                 if (p->next)
                 {
-                    block_output_io_buffer.reset(io_buffer_t::create(0));
+                    block_output_io_buffer.reset(io_buffer_t::create(STDOUT_FILENO));
                     if (block_output_io_buffer.get() == NULL)
                     {
                         /* We failed (e.g. no more fds could be created). */
@@ -1430,8 +1430,7 @@ void exec_job(parser_t &parser, job_t *j)
                 if (g_log_forks)
                 {
                     const wchar_t *file = reader_current_filename();
-                    const wchar_t *func = parser_t::principal_parser().is_function();
+                    printf("fork #%d: forking for '%s' in '%ls'\n", g_fork_count, actual_cmd, file ? file : L"");
-                    printf("fork #%d: forking for '%s' in '%ls:%ls'\n", g_fork_count, actual_cmd, file ? file : L"", func ? func : L"?");
 
                     fprintf(stderr, "IO chain for %s:\n", actual_cmd);
                     io_print(process_net_io_chain);
@@ -1606,7 +1605,7 @@ static int exec_subshell_internal(const wcstring &cmd, wcstring_list_t *lst, boo
     int subcommand_status = -1; //assume the worst
 
     // IO buffer creation may fail (e.g. if we have too many open files to make a pipe), so this may be null
-    const shared_ptr<io_buffer_t> io_buffer(io_buffer_t::create(0));
+    const shared_ptr<io_buffer_t> io_buffer(io_buffer_t::create(STDOUT_FILENO));
     if (io_buffer.get() != NULL)
     {
         parser_t &parser = parser_t::principal_parser();

expand.cpp

@@ -53,31 +53,6 @@ parameter expansion.
 
 #include "parse_util.h"
 
-/**
-   Error issued on invalid variable name
-*/
-#define COMPLETE_VAR_DESC _( L"The '$' character begins a variable name. The character '%lc', which directly followed a '$', is not allowed as a part of a variable name, and variable names may not be zero characters long. To learn more about variable expansion in fish, type 'help expand-variable'.")
-
-/**
-   Error issued on $?
-*/
-#define COMPLETE_YOU_WANT_STATUS _( L"$? is not a valid variable in fish. If you want the exit status of the last command, try $status.")
-
-/**
-   Error issued on invalid variable name
-*/
-#define COMPLETE_VAR_NULL_DESC _( L"The '$' begins a variable name. It was given at the end of an argument. Variable names may not be zero characters long. To learn more about variable expansion in fish, type 'help expand-variable'.")
-
-/**
-   Error issued on invalid variable name
-*/
-#define COMPLETE_VAR_BRACKET_DESC _( L"Did you mean %ls{$%ls}%ls? The '$' character begins a variable name. A bracket, which directly followed a '$', is not allowed as a part of a variable name, and variable names may not be zero characters long. To learn more about variable expansion in fish, type 'help expand-variable'." )
-
-/**
-   Error issued on invalid variable name
-*/
-#define COMPLETE_VAR_PARAN_DESC _( L"Did you mean (COMMAND)? In fish, the '$' character is only used for accessing variables. To learn more about command substitution in fish, type 'help expand-command-substitution'.")
-
 /**
    Description for child process
 */
@@ -156,6 +131,46 @@ int expand_is_clean(const wchar_t *in)
     return 1;
 }
 
+
+/* Append a syntax error to the given error list */
+static void append_syntax_error(parse_error_list_t *errors, size_t source_start, const wchar_t *fmt, ...)
+{
+    if (errors != NULL)
+    {
+        parse_error_t error;
+        error.source_start = source_start;
+        error.source_length = 0;
+        error.code = parse_error_syntax;
+
+        va_list va;
+        va_start(va, fmt);
+        error.text = vformat_string(fmt, va);
+        va_end(va);
+
+        errors->push_back(error);
+    }
+}
+
+/* Append a cmdsub error to the given error list */
+static void append_cmdsub_error(parse_error_list_t *errors, size_t source_start, const wchar_t *fmt, ...)
+{
+    if (errors != NULL)
+    {
+        parse_error_t error;
+        error.source_start = source_start;
+        error.source_length = 0;
+        error.code = parse_error_cmdsubst;
+
+        va_list va;
+        va_start(va, fmt);
+        error.text = vformat_string(fmt, va);
+        va_end(va);
+
+        errors->push_back(error);
+    }
+}
+
+
 /**
    Return the environment variable value for the string starting at \c in.
 */
@@ -859,7 +874,7 @@ static int expand_pid(const wcstring &instr_with_sep,
 }
 
 
-void expand_variable_error(parser_t &parser, const wcstring &token, size_t token_pos, int error_pos)
+void expand_variable_error(parser_t &parser, const wcstring &token, size_t token_pos, int error_pos, parse_error_list_t *errors)
 {
     size_t stop_pos = token_pos+1;
 
@@ -871,7 +886,7 @@ void expand_variable_error(parser_t &parser, const wcstring &token, size_t token
             *(cpy+token_pos)=0;
             wchar_t *name = &cpy[stop_pos+1];
             wchar_t *end = wcschr(name, BRACKET_END);
-            wchar_t *post;
+            wchar_t *post = NULL;
             int is_var=0;
             if (end)
             {
@@ -886,7 +901,7 @@ void expand_variable_error(parser_t &parser, const wcstring &token, size_t token
 
             if (is_var)
             {
-                parser.error(SYNTAX_ERROR,
+                append_syntax_error(errors,
                              error_pos,
                              COMPLETE_VAR_BRACKET_DESC,
                              cpy,
@@ -895,7 +910,7 @@ void expand_variable_error(parser_t &parser, const wcstring &token, size_t token
             }
             else
             {
-                parser.error(SYNTAX_ERROR,
+                append_syntax_error(errors,
                              error_pos,
                              COMPLETE_VAR_BRACKET_DESC,
                              L"",
@@ -909,7 +924,7 @@ void expand_variable_error(parser_t &parser, const wcstring &token, size_t token
 
         case INTERNAL_SEPARATOR:
         {
-            parser.error(SYNTAX_ERROR,
+            append_syntax_error(errors,
                          error_pos,
                          COMPLETE_VAR_PARAN_DESC);
             break;
@@ -917,7 +932,7 @@ void expand_variable_error(parser_t &parser, const wcstring &token, size_t token
 
         case 0:
         {
-            parser.error(SYNTAX_ERROR,
+            append_syntax_error(errors,
                          error_pos,
                          COMPLETE_VAR_NULL_DESC);
             break;
@@ -932,7 +947,7 @@ void expand_variable_error(parser_t &parser, const wcstring &token, size_t token
             else if (token_stop_char == ANY_STRING || token_stop_char == ANY_STRING_RECURSIVE)
                 token_stop_char = L'*';
 
-            parser.error(SYNTAX_ERROR,
+            append_syntax_error(errors,
                          error_pos,
                          (token_stop_char == L'?' ? COMPLETE_YOU_WANT_STATUS : COMPLETE_VAR_DESC),
                          token_stop_char);
@@ -944,7 +959,7 @@ void expand_variable_error(parser_t &parser, const wcstring &token, size_t token
 /**
    Parse an array slicing specification
  */
-static int parse_slice(const wchar_t *in, wchar_t **end_ptr, std::vector<long> &idx, size_t array_size)
+static int parse_slice(const wchar_t *in, wchar_t **end_ptr, std::vector<long> &idx, std::vector<size_t> &source_positions, size_t array_size)
 {
     wchar_t *end;
 
@@ -967,6 +982,7 @@ static int parse_slice(const wchar_t *in, wchar_t **end_ptr, std::vector<long> &
         }
 
         errno=0;
+        const size_t i1_src_pos = pos;
         tmp = wcstol(&in[pos], &end, 10);
         if ((errno) || (end == &in[pos]))
         {
@@ -983,6 +999,8 @@ static int parse_slice(const wchar_t *in, wchar_t **end_ptr, std::vector<long> &
             pos+=2;
             while (in[pos]==INTERNAL_SEPARATOR)
                 pos++;
+            
+            const size_t number_start = pos;
             long tmp1 = wcstol(&in[pos], &end, 10);
             if ((errno) || (end == &in[pos]))
             {
@@ -998,12 +1016,14 @@ static int parse_slice(const wchar_t *in, wchar_t **end_ptr, std::vector<long> &
             {
                 // debug(0, L"Expand range [subst]: %i\n", jjj);
                 idx.push_back(jjj);
+                source_positions.push_back(number_start);
             }
             continue;
         }
 
         // debug( 0, L"Push idx %d", tmp );
         idx.push_back(i1);
+        source_positions.push_back(i1_src_pos);
     }
 
     if (end_ptr)
@@ -1018,7 +1038,6 @@ static int parse_slice(const wchar_t *in, wchar_t **end_ptr, std::vector<long> &
 }
 
 
-
 /**
    Expand all environment variables in the string *ptr.
 
@@ -1033,24 +1052,29 @@ static int parse_slice(const wchar_t *in, wchar_t **end_ptr, std::vector<long> &
    happens, don't edit it unless you know exactly what you are doing,
    and do proper testing afterwards.
 */
-static int expand_variables_internal(parser_t &parser, wchar_t * const in, std::vector<completion_t> &out, long last_idx);
+static int expand_variables_internal(parser_t &parser, wchar_t * const in, std::vector<completion_t> &out, long last_idx, parse_error_list_t *errors);
 
-static int expand_variables2(parser_t &parser, const wcstring &instr, std::vector<completion_t> &out, long last_idx)
+static int expand_variables2(parser_t &parser, const wcstring &instr, std::vector<completion_t> &out, long last_idx, parse_error_list_t *errors)
 {
     wchar_t *in = wcsdup(instr.c_str());
-    int result = expand_variables_internal(parser, in, out, last_idx);
+    int result = expand_variables_internal(parser, in, out, last_idx, errors);
     free(in);
     return result;
 }
 
-static int expand_variables_internal(parser_t &parser, wchar_t * const in, std::vector<completion_t> &out, long last_idx)
+static int expand_variables_internal(parser_t &parser, wchar_t * const in, std::vector<completion_t> &out, long last_idx, parse_error_list_t *errors)
 {
     int is_ok= 1;
     int empty=0;
 
     wcstring var_tmp;
+    
+    // list of indexes
     std::vector<long> var_idx_list;
     
+    // parallel array of source positions of each index in the variable list
+    std::vector<size_t> var_pos_list;
+
     //  CHECK( out, 0 );
 
     for (long i=last_idx; (i>=0) && is_ok && !empty; i--)
@@ -1082,7 +1106,7 @@ static int expand_variables_internal(parser_t &parser, wchar_t * const in, std::
 
             if (var_len == 0)
             {
-                expand_variable_error(parser, in, stop_pos-1, -1);
+                expand_variable_error(parser, in, stop_pos-1, -1, errors);
 
                 is_ok = 0;
                 break;
@@ -1100,15 +1124,16 @@ static int expand_variables_internal(parser_t &parser, wchar_t * const in, std::
                 {
                     tokenize_variable_array(var_val.c_str(), var_item_list);
                     
-                    if (in[stop_pos] == L'[')
+                    const size_t slice_start = stop_pos;
+                    if (in[slice_start] == L'[')
                     {
                         wchar_t *slice_end;
                         all_vars=0;
 
-                        if (parse_slice(in + stop_pos, &slice_end, var_idx_list, var_item_list.size()))
+                        if (parse_slice(in + slice_start, &slice_end, var_idx_list, var_pos_list, var_item_list.size()))
                         {
-                            parser.error(SYNTAX_ERROR,
+                            append_syntax_error(errors,
-                                         -1,
+                                         stop_pos,
                                          L"Invalid index value");
                             is_ok = 0;
                             break;
@@ -1119,19 +1144,16 @@ static int expand_variables_internal(parser_t &parser, wchar_t * const in, std::
                     if (!all_vars)
                     {
                         wcstring_list_t string_values(var_idx_list.size());
-
                         for (size_t j=0; j<var_idx_list.size(); j++)
                         {
                             long tmp = var_idx_list.at(j);
-                            /*
+                            size_t var_src_pos = var_pos_list.at(j);
-                                           Check that we are within array
+                            /* Check that we are within array bounds. If not, truncate the list to exit. */
-                                           bounds. If not, truncate the list to
-                                           exit.
-                                           */
                             if (tmp < 1 || (size_t)tmp > var_item_list.size())
                             {
-                                parser.error(SYNTAX_ERROR,
+                                /* The slice was parsed starting at stop_pos, so we have to add that to the error position */
-                                             -1,
+                                append_syntax_error(errors,
+                                             slice_start + var_src_pos,
                                              ARRAY_BOUNDS_ERR);
                                 is_ok=0;
                                 var_idx_list.resize(j);
@@ -1170,7 +1192,7 @@ static int expand_variables_internal(parser_t &parser, wchar_t * const in, std::
                             }
                         }
                         res.append(in + stop_pos);
-                        is_ok &= expand_variables2(parser, res, out, i);
+                        is_ok &= expand_variables2(parser, res, out, i, errors);
                     }
                     else
                     {
@@ -1198,7 +1220,7 @@ static int expand_variables_internal(parser_t &parser, wchar_t * const in, std::
                                     }
                                     new_in.append(next);
                                     new_in.append(in + stop_pos);
-                                    is_ok &= expand_variables2(parser, new_in, out, i);
+                                    is_ok &= expand_variables2(parser, new_in, out, i, errors);
                                 }
                             }
 
@@ -1230,7 +1252,7 @@ static int expand_variables_internal(parser_t &parser, wchar_t * const in, std::
                     res.append(in);
                     res.append(in + stop_pos);
 
-                    is_ok &= expand_variables2(parser, res, out, i);
+                    is_ok &= expand_variables2(parser, res, out, i, errors);
                     return is_ok;
                 }
             }
@@ -1250,7 +1272,7 @@ static int expand_variables_internal(parser_t &parser, wchar_t * const in, std::
 /**
    Perform bracket expansion
 */
-static int expand_brackets(parser_t &parser, const wcstring &instr, int flags, std::vector<completion_t> &out)
+static int expand_brackets(parser_t &parser, const wcstring &instr, int flags, std::vector<completion_t> &out, parse_error_list_t *errors)
 {
     bool syntax_error = false;
     int bracket_count=0;
@@ -1320,14 +1342,14 @@ static int expand_brackets(parser_t &parser, const wcstring &instr, int flags, s
                 mod.push_back(BRACKET_END);
             }
 
-            return expand_brackets(parser, mod, 1, out);
+            return expand_brackets(parser, mod, 1, out, errors);
         }
     }
 
     if (syntax_error)
     {
-        parser.error(SYNTAX_ERROR,
+        append_syntax_error(errors,
-                     -1,
+                     SOURCE_LOCATION_UNKNOWN,
                      _(L"Mismatched brackets"));
         return 0;
     }
@@ -1356,7 +1378,7 @@ static int expand_brackets(parser_t &parser, const wcstring &instr, int flags, s
                 whole_item.append(in, length_preceding_brackets);
                 whole_item.append(item_begin, item_len);
                 whole_item.append(bracket_end + 1);
-                expand_brackets(parser, whole_item, flags, out);
+                expand_brackets(parser, whole_item, flags, out, errors);
 
                 item_begin = pos+1;
                 if (pos == bracket_end)
@@ -1380,7 +1402,7 @@ static int expand_brackets(parser_t &parser, const wcstring &instr, int flags, s
 /**
  Perform cmdsubst expansion
  */
-static int expand_cmdsubst(parser_t &parser, const wcstring &input, std::vector<completion_t> &out_list)
+static int expand_cmdsubst(parser_t &parser, const wcstring &input, std::vector<completion_t> &out_list, parse_error_list_t *errors)
 {
     wchar_t *paran_begin=0, *paran_end=0;
     std::vector<wcstring> sub_res;
@@ -1393,8 +1415,8 @@ static int expand_cmdsubst(parser_t &parser, const wcstring &input, std::vector<
     switch (parse_ret = parse_util_locate_cmdsubst(in, &paran_begin, &paran_end, false))
     {
         case -1:
-            parser.error(SYNTAX_ERROR,
+            append_syntax_error(errors,
-                         -1,
+                         SOURCE_LOCATION_UNKNOWN,
                          L"Mismatched parenthesis");
             return 0;
         case 0:
@@ -1409,7 +1431,7 @@ static int expand_cmdsubst(parser_t &parser, const wcstring &input, std::vector<
 
     if (exec_subshell(subcmd, sub_res, true /* do apply exit status */) == -1)
     {
-        parser.error(CMDSUBST_ERROR, -1, L"Unknown error while evaulating command substitution");
+        append_cmdsub_error(errors, SOURCE_LOCATION_UNKNOWN, L"Unknown error while evaulating command substitution");
         return 0;
     }
 
@@ -1417,24 +1439,25 @@ static int expand_cmdsubst(parser_t &parser, const wcstring &input, std::vector<
     if (*tail_begin == L'[')
     {
         std::vector<long> slice_idx;
+        std::vector<size_t> slice_source_positions;
         wchar_t *slice_end;
 
-        if (parse_slice(tail_begin, &slice_end, slice_idx, sub_res.size()))
+        if (parse_slice(tail_begin, &slice_end, slice_idx, slice_source_positions, sub_res.size()))
         {
-            parser.error(SYNTAX_ERROR, -1, L"Invalid index value");
+            append_syntax_error(errors, SOURCE_LOCATION_UNKNOWN, L"Invalid index value");
             return 0;
         }
         else
         {
-            std::vector<wcstring> sub_res2;
+            wcstring_list_t sub_res2;
             tail_begin = slice_end;
             for (i=0; i < slice_idx.size(); i++)
             {
                 long idx = slice_idx.at(i);
                 if (idx < 1 || (size_t)idx > sub_res.size())
                 {
-                    parser.error(SYNTAX_ERROR,
+                    append_syntax_error(errors,
-                                 -1,
+                                 SOURCE_LOCATION_UNKNOWN,
                                  ARRAY_BOUNDS_ERR);
                     return 0;
                 }
@@ -1455,7 +1478,7 @@ static int expand_cmdsubst(parser_t &parser, const wcstring &input, std::vector<
        of the string is inserted into the tail_expand array list
        */
     std::vector<completion_t> tail_expand;
-    expand_cmdsubst(parser, tail_begin, tail_expand);
+    expand_cmdsubst(parser, tail_begin, tail_expand, errors /* TODO: offset error locations */);
 
     /*
        Combine the result of the current command substitution with the
@@ -1664,7 +1687,7 @@ static void remove_internal_separator(wcstring &str, bool conv)
 }
 
 
-int expand_string(const wcstring &input, std::vector<completion_t> &output, expand_flags_t flags)
+int expand_string(const wcstring &input, std::vector<completion_t> &output, expand_flags_t flags, parse_error_list_t *errors)
 {
     parser_t parser(PARSER_TYPE_ERRORS_ONLY, true /* show errors */);
 
@@ -1686,14 +1709,14 @@ int expand_string(const wcstring &input, std::vector<completion_t> &output, expa
 
         if (parse_util_locate_cmdsubst(input.c_str(), &begin, &end, true) != 0)
         {
-            parser.error(CMDSUBST_ERROR, -1, L"Command substitutions not allowed");
+            append_cmdsub_error(errors, SOURCE_LOCATION_UNKNOWN, L"Command substitutions not allowed");
             return EXPAND_ERROR;
         }
         append_completion(*in, input);
     }
     else
     {
-        int cmdsubst_ok = expand_cmdsubst(parser, input, *in);
+        int cmdsubst_ok = expand_cmdsubst(parser, input, *in, errors);
         if (! cmdsubst_ok)
             return EXPAND_ERROR;
     }
@@ -1721,7 +1744,7 @@ int expand_string(const wcstring &input, std::vector<completion_t> &output, expa
         }
         else
         {
-            if (!expand_variables2(parser, next, *out, next.size() - 1))
+            if (!expand_variables2(parser, next, *out, next.size() - 1, errors))
             {
                 return EXPAND_ERROR;
             }
@@ -1735,7 +1758,7 @@ int expand_string(const wcstring &input, std::vector<completion_t> &output, expa
     {
         const wcstring &next = in->at(i).completion;
 
-        if (!expand_brackets(parser, next, flags, *out))
+        if (!expand_brackets(parser, next, flags, *out, errors))
         {
             return EXPAND_ERROR;
         }
@@ -1865,7 +1888,7 @@ int expand_string(const wcstring &input, std::vector<completion_t> &output, expa
     return res;
 }
 
-bool expand_one(wcstring &string, expand_flags_t flags)
+bool expand_one(wcstring &string, expand_flags_t flags, parse_error_list_t *errors)
 {
     std::vector<completion_t> completions;
     bool result = false;
@@ -1875,7 +1898,7 @@ bool expand_one(wcstring &string, expand_flags_t flags)
         return true;
     }
 
-    if (expand_string(string, completions, flags | EXPAND_NO_DESCRIPTIONS))
+    if (expand_string(string, completions, flags | EXPAND_NO_DESCRIPTIONS, errors))
     {
         if (completions.size() == 1)
         {

expand.h

@@ -19,6 +19,7 @@
 
 #include "util.h"
 #include "common.h"
+#include "parse_constants.h"
 #include <list>
 
 enum
@@ -146,9 +147,10 @@ class parser_t;
    \param input The parameter to expand
    \param output The list to which the result will be appended.
    \param flag Specifies if any expansion pass should be skipped. Legal values are any combination of EXPAND_SKIP_CMDSUBST EXPAND_SKIP_VARIABLES and EXPAND_SKIP_WILDCARDS
+   \param errors Resulting errors, or NULL to ignore
    \return One of EXPAND_OK, EXPAND_ERROR, EXPAND_WILDCARD_MATCH and EXPAND_WILDCARD_NO_MATCH. EXPAND_WILDCARD_NO_MATCH and EXPAND_WILDCARD_MATCH are normal exit conditions used only on strings containing wildcards to tell if the wildcard produced any matches.
 */
-__warn_unused int expand_string(const wcstring &input, std::vector<completion_t> &output, expand_flags_t flags);
+__warn_unused int expand_string(const wcstring &input, std::vector<completion_t> &output, expand_flags_t flags, parse_error_list_t *errors);
 
 
 /**
@@ -158,9 +160,10 @@ __warn_unused int expand_string(const wcstring &input, std::vector<completion_t>
 
    \param inout_str The parameter to expand in-place
    \param flag Specifies if any expansion pass should be skipped. Legal values are any combination of EXPAND_SKIP_CMDSUBST EXPAND_SKIP_VARIABLES and EXPAND_SKIP_WILDCARDS
+   \param errors Resulting errors, or NULL to ignore
    \return Whether expansion succeded
 */
-bool expand_one(wcstring &inout_str, expand_flags_t flags);
+bool expand_one(wcstring &inout_str, expand_flags_t flags, parse_error_list_t *errors = NULL);
 
 /**
    Convert the variable value to a human readable form, i.e. escape things, handle arrays, etc. Suitable for pretty-printing. The result must be free'd!

fish.cpp

@@ -73,6 +73,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 */
 #define GETOPT_STRING "+hilnvc:p:d:"
 
+/* If we are doing profiling, the filename to output to */
+static const char *s_profiling_output_filename = NULL;
+
 static bool has_suffix(const std::string &path, const char *suffix, bool ignore_case)
 {
     size_t pathlen = path.size(), suffixlen = strlen(suffix);
@@ -224,7 +227,9 @@ static void source_config_in_directory(const wcstring &dir)
     const wcstring escaped_dir = escape_string(dir, ESCAPE_ALL);
     const wcstring cmd = L"builtin source " + escaped_dir + L"/config.fish 2>/dev/null";
     parser_t &parser = parser_t::principal_parser();
+    parser.set_is_within_fish_initialization(true);
     parser.eval(cmd, io_chain_t(), TOP);
+    parser.set_is_within_fish_initialization(false);
 }
 
 /**
@@ -245,7 +250,6 @@ static int read_init(const struct config_paths_t &paths)
     return 1;
 }
 
-
 /**
   Parse the argument list, return the index of the first non-switch
   arguments.
@@ -346,7 +350,8 @@ static int fish_parse_opt(int argc, char **argv, std::vector<std::string> *out_c
 
             case 'p':
             {
-                profile = optarg;
+                s_profiling_output_filename = optarg;
+                g_profiling_active = true;
                 break;
             }
 
@@ -521,11 +526,16 @@ int main(int argc, char **argv)
 
     restore_term_mode();
     restore_term_foreground_process_group();
+    
+    if (g_profiling_active)
+    {
+        parser.emit_profiling(s_profiling_output_filename);
+    }
+    
     history_destroy();
     proc_destroy();
     builtin_destroy();
     reader_destroy();
-    parser.destroy();
     wutil_destroy();
     event_destroy();
 

fish.xcodeproj/project.pbxproj

@@ -18,7 +18,6 @@
 			dependencies = (
 				D07D265715E33B86009E43F6 /* PBXTargetDependency */,
 				D07D265915E33B86009E43F6 /* PBXTargetDependency */,
-				D07D265B15E33B86009E43F6 /* PBXTargetDependency */,
 				D07D265D15E33B86009E43F6 /* PBXTargetDependency */,
 				D0A56500168D257900AF6161 /* PBXTargetDependency */,
 			);
@@ -49,7 +48,6 @@
 			dependencies = (
 				D0F01A1315AA36280034B3B1 /* PBXTargetDependency */,
 				D0F01A1515AA362E0034B3B1 /* PBXTargetDependency */,
-				D0F01A1915AA36310034B3B1 /* PBXTargetDependency */,
 				D0F01A1715AA36300034B3B1 /* PBXTargetDependency */,
 				D0A564EF168D09C000AF6161 /* PBXTargetDependency */,
 			);
@@ -62,6 +60,7 @@
 		D01A2D24169B736200767098 /* man1 in Copy Files */ = {isa = PBXBuildFile; fileRef = D01A2D23169B730A00767098 /* man1 */; };
 		D01A2D25169B737700767098 /* man1 in CopyFiles */ = {isa = PBXBuildFile; fileRef = D01A2D23169B730A00767098 /* man1 */; };
 		D031890C15E36E4600D9CC39 /* base in Resources */ = {isa = PBXBuildFile; fileRef = D031890915E36D9800D9CC39 /* base */; };
+		D032388B1849D1980032CF2C /* pager.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D03238891849D1980032CF2C /* pager.cpp */; };
 		D033781115DC6D4C00A634BA /* completions in CopyFiles */ = {isa = PBXBuildFile; fileRef = D025C02715D1FEA100B9DB63 /* completions */; };
 		D033781215DC6D5200A634BA /* functions in CopyFiles */ = {isa = PBXBuildFile; fileRef = D025C02815D1FEA100B9DB63 /* functions */; };
 		D033781315DC6D5400A634BA /* tools in CopyFiles */ = {isa = PBXBuildFile; fileRef = D025C02915D1FEA100B9DB63 /* tools */; };
@@ -114,10 +113,11 @@
 		D08A32B917B446B100F3A533 /* parse_productions.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */; };
 		D08A32BA17B446B100F3A533 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; };
 		D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
-		D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
 		D0A564FE168D23D800AF6161 /* man in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; };
 		D0A56501168D258300AF6161 /* man in Copy Files */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; };
 		D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; };
+		D0C9733818DE5449002D7C81 /* utf8.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C9733718DE5449002D7C81 /* utf8.cpp */; };
+		D0C9733918DE5449002D7C81 /* utf8.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C9733718DE5449002D7C81 /* utf8.cpp */; };
 		D0CBD587159EF0E10024809C /* launch_fish.scpt in Resources */ = {isa = PBXBuildFile; fileRef = D0CBD586159EF0E10024809C /* launch_fish.scpt */; };
 		D0D02A67159837AD008E62BD /* complete.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853713B3ACEE0099B651 /* complete.cpp */; };
 		D0D02A69159837B2008E62BD /* env.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853A13B3ACEE0099B651 /* env.cpp */; };
@@ -153,7 +153,6 @@
 		D0D02A87159839D5008E62BD /* screen.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855A13B3ACEE0099B651 /* screen.cpp */; };
 		D0D02A88159839D5008E62BD /* signal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855C13B3ACEE0099B651 /* signal.cpp */; };
 		D0D02A89159839DF008E62BD /* fish.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854213B3ACEE0099B651 /* fish.cpp */; };
-		D0D02A8B15983CDF008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
 		D0D02A8D15983CFA008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
 		D0D02A8F15983D8F008E62BD /* parser_keywords.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855313B3ACEE0099B651 /* parser_keywords.cpp */; };
 		D0D02AC215985F3F008E62BD /* fishd.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854313B3ACEE0099B651 /* fishd.cpp */; };
@@ -162,7 +161,6 @@
 		D0D02AC515985F5B008E62BD /* print_help.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855613B3ACEE0099B651 /* print_help.cpp */; };
 		D0D02AC615985F65008E62BD /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; };
 		D0D02AC715985F9D008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
-		D0D02AC815985F9F008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
 		D0D02AD615986492008E62BD /* fish_indent.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853F13B3ACEE0099B651 /* fish_indent.cpp */; };
 		D0D02AD715986498008E62BD /* print_help.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855613B3ACEE0099B651 /* print_help.cpp */; };
 		D0D02AD81598649E008E62BD /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; };
@@ -170,25 +168,11 @@
 		D0D02ADA159864AB008E62BD /* wutil.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0856113B3ACEE0099B651 /* wutil.cpp */; };
 		D0D02ADB159864C2008E62BD /* tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855D13B3ACEE0099B651 /* tokenizer.cpp */; };
 		D0D02ADC159864D5008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
-		D0D02ADD159864D7008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
-		D0D02AEA15986549008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
-		D0D02AEB1598654C008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
-		D0D1CD6C15B7451900F03988 /* fish_pager.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854013B3ACEE0099B651 /* fish_pager.cpp */; };
-		D0D1CD6D15B7452100F03988 /* output.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855113B3ACEE0099B651 /* output.cpp */; };
-		D0D1CD6E15B7452600F03988 /* wutil.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0856113B3ACEE0099B651 /* wutil.cpp */; };
-		D0D1CD6F15B7452D00F03988 /* input_common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854913B3ACEE0099B651 /* input_common.cpp */; };
-		D0D1CD7015B7453300F03988 /* env_universal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853913B3ACEE0099B651 /* env_universal.cpp */; };
-		D0D1CD7115B7453700F03988 /* env_universal_common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853813B3ACEE0099B651 /* env_universal_common.cpp */; };
-		D0D1CD7215B7454A00F03988 /* print_help.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855613B3ACEE0099B651 /* print_help.cpp */; };
-		D0D1CD7315B7455200F03988 /* color.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0B6B0FE14E88BA400AD6C10 /* color.cpp */; };
-		D0D1CD7415B7456000F03988 /* iothread.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854D13B3ACEE0099B651 /* iothread.cpp */; };
-		D0D1CD7515B7458B00F03988 /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; };
 		D0D2694915983772005D9B9C /* function.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854413B3ACEE0099B651 /* function.cpp */; };
 		D0D2694A15983779005D9B9C /* builtin.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853513B3ACEE0099B651 /* builtin.cpp */; };
 		D0F019F115A977140034B3B1 /* fish in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0D2693C159835CA005D9B9C /* fish */; };
 		D0F019F215A977270034B3B1 /* fishd in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0D02ABC15985EF9008E62BD /* fishd */; };
 		D0F019F315A977290034B3B1 /* fish_indent in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0D02AD01598642A008E62BD /* fish_indent */; };
-		D0F019F415A9772C0034B3B1 /* fish_pager in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0D02AE415986537008E62BD /* fish_pager */; };
 		D0F019F815A977AB0034B3B1 /* config.fish in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0CBD580159EE48F0024809C /* config.fish */; };
 		D0F019FD15A977CA0034B3B1 /* config.fish in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0C4FD9415A7D7EE00212EF1 /* config.fish */; };
 		D0F01A0315A978910034B3B1 /* osx_fish_launcher.m in Sources */ = {isa = PBXBuildFile; fileRef = D0D02AFA159871B2008E62BD /* osx_fish_launcher.m */; };
@@ -218,13 +202,6 @@
 			remoteGlobalIDString = D0D02ABB15985EF9008E62BD;
 			remoteInfo = fishd;
 		};
-		D07D265C15E33B86009E43F6 /* PBXContainerItemProxy */ = {
-			isa = PBXContainerItemProxy;
-			containerPortal = D0A084F213B3AC130099B651 /* Project object */;
-			proxyType = 1;
-			remoteGlobalIDString = D0D02AE315986537008E62BD;
-			remoteInfo = fish_pager;
-		};
 		D07D265E15E33B86009E43F6 /* PBXContainerItemProxy */ = {
 			isa = PBXContainerItemProxy;
 			containerPortal = D0A084F213B3AC130099B651 /* Project object */;
@@ -267,13 +244,6 @@
 			remoteGlobalIDString = D0D02ACF1598642A008E62BD;
 			remoteInfo = fish_indent;
 		};
-		D0F01A1815AA36310034B3B1 /* PBXContainerItemProxy */ = {
-			isa = PBXContainerItemProxy;
-			containerPortal = D0A084F213B3AC130099B651 /* Project object */;
-			proxyType = 1;
-			remoteGlobalIDString = D0D02AE315986537008E62BD;
-			remoteInfo = fish_pager;
-		};
 /* End PBXContainerItemProxy section */
 
 /* Begin PBXCopyFilesBuildPhase section */
@@ -354,7 +324,6 @@
 				D0F019F115A977140034B3B1 /* fish in CopyFiles */,
 				D0F019F215A977270034B3B1 /* fishd in CopyFiles */,
 				D0F019F315A977290034B3B1 /* fish_indent in CopyFiles */,
-				D0F019F415A9772C0034B3B1 /* fish_pager in CopyFiles */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -386,6 +355,8 @@
 		D025C02815D1FEA100B9DB63 /* functions */ = {isa = PBXFileReference; lastKnownFileType = folder; name = functions; path = share/functions; sourceTree = "<group>"; };
 		D025C02915D1FEA100B9DB63 /* tools */ = {isa = PBXFileReference; lastKnownFileType = folder; name = tools; path = share/tools; sourceTree = "<group>"; };
 		D031890915E36D9800D9CC39 /* base */ = {isa = PBXFileReference; lastKnownFileType = text; path = base; sourceTree = BUILT_PRODUCTS_DIR; };
+		D03238891849D1980032CF2C /* pager.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = pager.cpp; sourceTree = "<group>"; };
+		D032388A1849D1980032CF2C /* pager.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pager.h; sourceTree = "<group>"; };
 		D03EE83814DF88B200FC7150 /* lru.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = lru.h; sourceTree = "<group>"; };
 		D052D8091868F7FC003ABCBD /* parse_execution.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_execution.cpp; sourceTree = "<group>"; };
 		D052D80A1868F7FC003ABCBD /* parse_execution.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_execution.h; sourceTree = "<group>"; };
@@ -455,7 +426,6 @@
 		D0A0853D13B3ACEE0099B651 /* expand.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = expand.cpp; sourceTree = "<group>"; };
 		D0A0853E13B3ACEE0099B651 /* fallback.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = fallback.cpp; sourceTree = "<group>"; };
 		D0A0853F13B3ACEE0099B651 /* fish_indent.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = fish_indent.cpp; sourceTree = "<group>"; };
-		D0A0854013B3ACEE0099B651 /* fish_pager.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = fish_pager.cpp; sourceTree = "<group>"; };
 		D0A0854113B3ACEE0099B651 /* fish_tests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = fish_tests.cpp; sourceTree = "<group>"; };
 		D0A0854213B3ACEE0099B651 /* fish.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = fish.cpp; sourceTree = "<group>"; };
 		D0A0854313B3ACEE0099B651 /* fishd.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = fishd.cpp; sourceTree = "<group>"; };
@@ -503,17 +473,17 @@
 		D0C6FCC914CFA4B0004CE8AD /* autoload.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = autoload.cpp; sourceTree = "<group>"; };
 		D0C6FCCB14CFA4B7004CE8AD /* autoload.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = autoload.h; sourceTree = "<group>"; };
 		D0C861EA16CC7054003B5A04 /* builtin_set_color.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_set_color.cpp; sourceTree = "<group>"; };
+		D0C9733718DE5449002D7C81 /* utf8.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = utf8.cpp; sourceTree = "<group>"; };
+		D0C9733A18DE5451002D7C81 /* utf8.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = utf8.h; sourceTree = "<group>"; };
 		D0CA63F316FC275F00093BD4 /* builtin_printf.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_printf.cpp; sourceTree = "<group>"; };
 		D0CBD580159EE48F0024809C /* config.fish */ = {isa = PBXFileReference; lastKnownFileType = text; name = config.fish; path = share/config.fish; sourceTree = "<group>"; };
 		D0CBD583159EEE010024809C /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = System/Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; };
 		D0CBD586159EF0E10024809C /* launch_fish.scpt */ = {isa = PBXFileReference; lastKnownFileType = file; name = launch_fish.scpt; path = osx/launch_fish.scpt; sourceTree = "<group>"; };
-		D0D02A8A15983CDF008E62BD /* libiconv.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libiconv.dylib; path = usr/lib/libiconv.dylib; sourceTree = SDKROOT; };
 		D0D02A8C15983CFA008E62BD /* libncurses.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libncurses.dylib; path = usr/lib/libncurses.dylib; sourceTree = SDKROOT; };
 		D0D02A9A15985A75008E62BD /* fish.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = fish.app; sourceTree = BUILT_PRODUCTS_DIR; };
 		D0D02AA915985C0C008E62BD /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = Info.plist; path = osx/Info.plist; sourceTree = "<group>"; };
 		D0D02ABC15985EF9008E62BD /* fishd */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fishd; sourceTree = BUILT_PRODUCTS_DIR; };
 		D0D02AD01598642A008E62BD /* fish_indent */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fish_indent; sourceTree = BUILT_PRODUCTS_DIR; };
-		D0D02AE415986537008E62BD /* fish_pager */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fish_pager; sourceTree = BUILT_PRODUCTS_DIR; };
 		D0D02AFA159871B2008E62BD /* osx_fish_launcher.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = osx_fish_launcher.m; path = osx/osx_fish_launcher.m; sourceTree = "<group>"; };
 		D0D2693C159835CA005D9B9C /* fish */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fish; sourceTree = BUILT_PRODUCTS_DIR; };
 		D0D9B2B318555D92001AE279 /* parse_constants.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = parse_constants.h; sourceTree = "<group>"; };
@@ -528,7 +498,6 @@
 			isa = PBXFrameworksBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
-				D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */,
 				D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -538,7 +507,6 @@
 			buildActionMask = 2147483647;
 			files = (
 				D0D02AC715985F9D008E62BD /* libncurses.dylib in Frameworks */,
-				D0D02AC815985F9F008E62BD /* libiconv.dylib in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -547,16 +515,6 @@
 			buildActionMask = 2147483647;
 			files = (
 				D0D02ADC159864D5008E62BD /* libncurses.dylib in Frameworks */,
-				D0D02ADD159864D7008E62BD /* libiconv.dylib in Frameworks */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-		D0D02AE115986537008E62BD /* Frameworks */ = {
-			isa = PBXFrameworksBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				D0D02AEA15986549008E62BD /* libncurses.dylib in Frameworks */,
-				D0D02AEB1598654C008E62BD /* libiconv.dylib in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -565,7 +523,6 @@
 			buildActionMask = 2147483647;
 			files = (
 				D0D02A8D15983CFA008E62BD /* libncurses.dylib in Frameworks */,
-				D0D02A8B15983CDF008E62BD /* libiconv.dylib in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -621,7 +578,6 @@
 			isa = PBXGroup;
 			children = (
 				D0D02A8C15983CFA008E62BD /* libncurses.dylib */,
-				D0D02A8A15983CDF008E62BD /* libiconv.dylib */,
 				D0CBD583159EEE010024809C /* Foundation.framework */,
 			);
 			name = Libraries;
@@ -673,7 +629,6 @@
 				D0A0850E13B3ACEE0099B651 /* function.h */,
 				D0A0854413B3ACEE0099B651 /* function.cpp */,
 				D0A0853F13B3ACEE0099B651 /* fish_indent.cpp */,
-				D0A0854013B3ACEE0099B651 /* fish_pager.cpp */,
 				D0A0854113B3ACEE0099B651 /* fish_tests.cpp */,
 				D0A0854213B3ACEE0099B651 /* fish.cpp */,
 				D0A0854313B3ACEE0099B651 /* fishd.cpp */,
@@ -699,6 +654,8 @@
 				D0A0855013B3ACEE0099B651 /* mimedb.cpp */,
 				D0A0851A13B3ACEE0099B651 /* output.h */,
 				D0A0855113B3ACEE0099B651 /* output.cpp */,
+				D032388A1849D1980032CF2C /* pager.h */,
+				D03238891849D1980032CF2C /* pager.cpp */,
 				D0A0851B13B3ACEE0099B651 /* parse_util.h */,
 				D0A0855213B3ACEE0099B651 /* parse_util.cpp */,
 				D0A0851C13B3ACEE0099B651 /* parser_keywords.h */,
@@ -723,6 +680,8 @@
 				D0A0855C13B3ACEE0099B651 /* signal.cpp */,
 				D0A0852513B3ACEE0099B651 /* tokenizer.h */,
 				D0A0855D13B3ACEE0099B651 /* tokenizer.cpp */,
+				D0C9733A18DE5451002D7C81 /* utf8.h */,
+				D0C9733718DE5449002D7C81 /* utf8.cpp */,
 				D0A0852613B3ACEE0099B651 /* util.h */,
 				D0A0855E13B3ACEE0099B651 /* util.cpp */,
 				D0A0852713B3ACEE0099B651 /* wgetopt.h */,
@@ -784,7 +743,6 @@
 				D0D02A9A15985A75008E62BD /* fish.app */,
 				D0D02ABC15985EF9008E62BD /* fishd */,
 				D0D02AD01598642A008E62BD /* fish_indent */,
-				D0D02AE415986537008E62BD /* fish_pager */,
 				D08A328D17B4455100F3A533 /* fish_tests */,
 			);
 			name = Products;
@@ -885,22 +843,6 @@
 			productReference = D0D02AD01598642A008E62BD /* fish_indent */;
 			productType = "com.apple.product-type.tool";
 		};
-		D0D02AE315986537008E62BD /* fish_pager */ = {
-			isa = PBXNativeTarget;
-			buildConfigurationList = D0D02AE715986537008E62BD /* Build configuration list for PBXNativeTarget "fish_pager" */;
-			buildPhases = (
-				D0D02AE015986537008E62BD /* Sources */,
-				D0D02AE115986537008E62BD /* Frameworks */,
-			);
-			buildRules = (
-			);
-			dependencies = (
-			);
-			name = fish_pager;
-			productName = fish_pager;
-			productReference = D0D02AE415986537008E62BD /* fish_pager */;
-			productType = "com.apple.product-type.tool";
-		};
 		D0D2693B159835CA005D9B9C /* fish_shell */ = {
 			isa = PBXNativeTarget;
 			buildConfigurationList = D0D26943159835CA005D9B9C /* Build configuration list for PBXNativeTarget "fish_shell" */;
@@ -923,7 +865,7 @@
 		D0A084F213B3AC130099B651 /* Project object */ = {
 			isa = PBXProject;
 			attributes = {
-				LastUpgradeCheck = 0460;
+				LastUpgradeCheck = 0500;
 			};
 			buildConfigurationList = D0A084F513B3AC130099B651 /* Build configuration list for PBXProject "fish" */;
 			compatibilityVersion = "Xcode 3.2";
@@ -943,7 +885,6 @@
 				D0D2693B159835CA005D9B9C /* fish_shell */,
 				D0D02ABB15985EF9008E62BD /* fishd */,
 				D0D02ACF1598642A008E62BD /* fish_indent */,
-				D0D02AE315986537008E62BD /* fish_pager */,
 				D08A328C17B4455100F3A533 /* fish_tests */,
 				D0A564E6168CFDD800AF6161 /* man_pages */,
 				D0A084F713B3AC130099B651 /* Makefile */,
@@ -1003,7 +944,6 @@
 				"$(SRCROOT)/doc_src/fish.txt",
 				"$(SRCROOT)/doc_src/fish_config.txt",
 				"$(SRCROOT)/doc_src/fish_indent.txt",
-				"$(SRCROOT)/doc_src/fish_pager.txt",
 				"$(SRCROOT)/doc_src/fish_prompt.txt",
 				"$(SRCROOT)/doc_src/fish_right_prompt.txt",
 				"$(SRCROOT)/doc_src/fish_update_completions.txt",
@@ -1076,7 +1016,6 @@
 				"$(BUILT_PRODUCTS_DIR)/man/man1/fish.1",
 				"$(BUILT_PRODUCTS_DIR)/man/man1/fish_config.1",
 				"$(BUILT_PRODUCTS_DIR)/man/man1/fish_indent.1",
-				"$(BUILT_PRODUCTS_DIR)/man/man1/fish_pager.1",
 				"$(BUILT_PRODUCTS_DIR)/man/man1/fish_prompt.1",
 				"$(BUILT_PRODUCTS_DIR)/man/man1/fish_right_prompt.1",
 				"$(BUILT_PRODUCTS_DIR)/man/man1/fish_update_completions.1",
@@ -1120,7 +1059,7 @@
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 			shellPath = /bin/sh;
-			shellScript = "cd \"${SRCROOT}\" ;\n# Run build_documentation.sh\n# Do this in a subshell so that we keep going even if it calls exit\n( . \"./build_tools/build_documentation.sh\" \"./Doxyfile.help\" \"./doc_src\" \"$BUILT_PRODUCTS_DIR\" )\n\n# Copy certain files into man1, destined for share/man/man1 (instead of share/fish/man/man1)\n# These copies will fail of the documentation did not build; that's OK\n# We want to create the directory even if the documentation did not build, so that the Xcode build can still succeed\nmanpathdir=\"${BUILT_PRODUCTS_DIR}/pages_for_manpath/man1\"\necho \"Copying pages destined for manpath into $manpathdir\"\nrm -Rf \"$manpathdir\"\nmkdir -p \"$manpathdir\"\nfor manpage in fish.1 set_color.1 fish_pager.1 fishd.1 fish_indent.1; do\n  manpagepath=\"${BUILT_PRODUCTS_DIR}/man/man1/${manpage}\"\n  test -f \"$manpagepath\" && cp \"$manpagepath\" \"${BUILT_PRODUCTS_DIR}/pages_for_manpath/man1/\"\ndone\n\n# Always succeed\ntrue\n";
+			shellScript = "cd \"${SRCROOT}\" ;\n# Run build_documentation.sh\n# Do this in a subshell so that we keep going even if it calls exit\n( . \"./build_tools/build_documentation.sh\" \"./Doxyfile.help\" \"./doc_src\" \"$BUILT_PRODUCTS_DIR\" )\n\n# Copy certain files into man1, destined for share/man/man1 (instead of share/fish/man/man1)\n# These copies will fail of the documentation did not build; that's OK\n# We want to create the directory even if the documentation did not build, so that the Xcode build can still succeed\nmanpathdir=\"${BUILT_PRODUCTS_DIR}/pages_for_manpath/man1\"\necho \"Copying pages destined for manpath into $manpathdir\"\nrm -Rf \"$manpathdir\"\nmkdir -p \"$manpathdir\"\nfor manpage in fish.1 set_color.1 fishd.1 fish_indent.1; do\n  manpagepath=\"${BUILT_PRODUCTS_DIR}/man/man1/${manpage}\"\n  test -f \"$manpagepath\" && cp \"$manpagepath\" \"${BUILT_PRODUCTS_DIR}/pages_for_manpath/man1/\"\ndone\n\n# Always succeed\ntrue\n";
 		};
 /* End PBXShellScriptBuildPhase section */
 
@@ -1177,6 +1116,7 @@
 			files = (
 				D0D02AC215985F3F008E62BD /* fishd.cpp in Sources */,
 				D0D02AC315985F43008E62BD /* env_universal_common.cpp in Sources */,
+				D0C9733918DE5449002D7C81 /* utf8.cpp in Sources */,
 				D0D02AC415985F4D008E62BD /* wutil.cpp in Sources */,
 				D0D02AC515985F5B008E62BD /* print_help.cpp in Sources */,
 				D0D02AC615985F65008E62BD /* common.cpp in Sources */,
@@ -1196,23 +1136,6 @@
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
-		D0D02AE015986537008E62BD /* Sources */ = {
-			isa = PBXSourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				D0D1CD6C15B7451900F03988 /* fish_pager.cpp in Sources */,
-				D0D1CD6D15B7452100F03988 /* output.cpp in Sources */,
-				D0D1CD6E15B7452600F03988 /* wutil.cpp in Sources */,
-				D0D1CD6F15B7452D00F03988 /* input_common.cpp in Sources */,
-				D0D1CD7015B7453300F03988 /* env_universal.cpp in Sources */,
-				D0D1CD7115B7453700F03988 /* env_universal_common.cpp in Sources */,
-				D0D1CD7315B7455200F03988 /* color.cpp in Sources */,
-				D0D1CD7515B7458B00F03988 /* common.cpp in Sources */,
-				D0D1CD7215B7454A00F03988 /* print_help.cpp in Sources */,
-				D0D1CD7415B7456000F03988 /* iothread.cpp in Sources */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
 		D0D26938159835CA005D9B9C /* Sources */ = {
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
@@ -1231,6 +1154,7 @@
 				D0D02A86159839D5008E62BD /* postfork.cpp in Sources */,
 				D0D02A87159839D5008E62BD /* screen.cpp in Sources */,
 				D0D02A88159839D5008E62BD /* signal.cpp in Sources */,
+				D0C9733818DE5449002D7C81 /* utf8.cpp in Sources */,
 				D0D2694A15983779005D9B9C /* builtin.cpp in Sources */,
 				D0D2694915983772005D9B9C /* function.cpp in Sources */,
 				D0D02A67159837AD008E62BD /* complete.cpp in Sources */,
@@ -1254,6 +1178,7 @@
 				D0D02A7915983888008E62BD /* intern.cpp in Sources */,
 				D0D02A7A15983916008E62BD /* env_universal.cpp in Sources */,
 				D0D02A7B15983928008E62BD /* env_universal_common.cpp in Sources */,
+				D032388B1849D1980032CF2C /* pager.cpp in Sources */,
 				D0D02A89159839DF008E62BD /* fish.cpp in Sources */,
 				D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */,
 				D0FE8EE8179FB760008C9F21 /* parse_productions.cpp in Sources */,
@@ -1286,11 +1211,6 @@
 			target = D0D02ABB15985EF9008E62BD /* fishd */;
 			targetProxy = D07D265A15E33B86009E43F6 /* PBXContainerItemProxy */;
 		};
-		D07D265B15E33B86009E43F6 /* PBXTargetDependency */ = {
-			isa = PBXTargetDependency;
-			target = D0D02AE315986537008E62BD /* fish_pager */;
-			targetProxy = D07D265C15E33B86009E43F6 /* PBXContainerItemProxy */;
-		};
 		D07D265D15E33B86009E43F6 /* PBXTargetDependency */ = {
 			isa = PBXTargetDependency;
 			target = D0D02ACF1598642A008E62BD /* fish_indent */;
@@ -1321,18 +1241,12 @@
 			target = D0D02ACF1598642A008E62BD /* fish_indent */;
 			targetProxy = D0F01A1615AA36300034B3B1 /* PBXContainerItemProxy */;
 		};
-		D0F01A1915AA36310034B3B1 /* PBXTargetDependency */ = {
-			isa = PBXTargetDependency;
-			target = D0D02AE315986537008E62BD /* fish_pager */;
-			targetProxy = D0F01A1815AA36310034B3B1 /* PBXContainerItemProxy */;
-		};
 /* End PBXTargetDependency section */
 
 /* Begin XCBuildConfiguration section */
 		D007FDDA17136EAA00A52BE6 /* Release_C++11 */ = {
 			isa = XCBuildConfiguration;
 			buildSettings = {
-				ARCHS = "$(ARCHS_STANDARD_32_64_BIT)";
 				CLANG_CXX_LANGUAGE_STANDARD = "c++0x";
 				CLANG_CXX_LIBRARY = "libc++";
 				DEAD_CODE_STRIPPING = YES;
@@ -1430,18 +1344,6 @@
 			};
 			name = "Release_C++11";
 		};
-		D007FDE117136EAA00A52BE6 /* Release_C++11 */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				COPY_PHASE_STRIP = YES;
-				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
-				GCC_ENABLE_OBJC_EXCEPTIONS = YES;
-				GCC_WARN_UNINITIALIZED_AUTOS = YES;
-				PRODUCT_NAME = "$(TARGET_NAME)";
-			};
-			name = "Release_C++11";
-		};
 		D007FDE217136EAA00A52BE6 /* Release_C++11 */ = {
 			isa = XCBuildConfiguration;
 			buildSettings = {
@@ -1488,7 +1390,6 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				ALWAYS_SEARCH_USER_PATHS = NO;
-				ARCHS = "$(ARCHS_STANDARD_64_BIT)";
 				CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_WARN_CONSTANT_CONVERSION = YES;
@@ -1514,7 +1415,6 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				ALWAYS_SEARCH_USER_PATHS = NO;
-				ARCHS = "$(ARCHS_STANDARD_64_BIT)";
 				CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_WARN_CONSTANT_CONVERSION = YES;
@@ -1535,7 +1435,6 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				ALWAYS_SEARCH_USER_PATHS = NO;
-				ARCHS = "$(ARCHS_STANDARD_64_BIT)";
 				CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_WARN_CONSTANT_CONVERSION = YES;
@@ -1555,7 +1454,6 @@
 		D0A084F813B3AC130099B651 /* Debug */ = {
 			isa = XCBuildConfiguration;
 			buildSettings = {
-				ARCHS = "$(ARCHS_STANDARD_32_64_BIT)";
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				GCC_ENABLE_CPP_EXCEPTIONS = NO;
 				GCC_ENABLE_CPP_RTTI = YES;
@@ -1584,7 +1482,6 @@
 		D0A084F913B3AC130099B651 /* Release */ = {
 			isa = XCBuildConfiguration;
 			buildSettings = {
-				ARCHS = "$(ARCHS_STANDARD_32_64_BIT)";
 				DEAD_CODE_STRIPPING = YES;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				GCC_ENABLE_CPP_EXCEPTIONS = NO;
@@ -1743,34 +1640,6 @@
 			};
 			name = Release;
 		};
-		D0D02AE815986537008E62BD /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				COPY_PHASE_STRIP = NO;
-				GCC_DYNAMIC_NO_PIC = NO;
-				GCC_ENABLE_OBJC_EXCEPTIONS = YES;
-				GCC_PREPROCESSOR_DEFINITIONS = (
-					"DEBUG=1",
-					"$(inherited)",
-				);
-				GCC_WARN_UNINITIALIZED_AUTOS = YES;
-				PRODUCT_NAME = "$(TARGET_NAME)";
-			};
-			name = Debug;
-		};
-		D0D02AE915986537008E62BD /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				COPY_PHASE_STRIP = YES;
-				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
-				GCC_ENABLE_OBJC_EXCEPTIONS = YES;
-				GCC_WARN_UNINITIALIZED_AUTOS = YES;
-				PRODUCT_NAME = "$(TARGET_NAME)";
-			};
-			name = Release;
-		};
 		D0D26944159835CA005D9B9C /* Debug */ = {
 			isa = XCBuildConfiguration;
 			buildSettings = {
@@ -1898,16 +1767,6 @@
 			defaultConfigurationIsVisible = 0;
 			defaultConfigurationName = Release;
 		};
-		D0D02AE715986537008E62BD /* Build configuration list for PBXNativeTarget "fish_pager" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				D0D02AE815986537008E62BD /* Debug */,
-				D0D02AE915986537008E62BD /* Release */,
-				D007FDE117136EAA00A52BE6 /* Release_C++11 */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
 		D0D26943159835CA005D9B9C /* Build configuration list for PBXNativeTarget "fish_shell" */ = {
 			isa = XCConfigurationList;
 			buildConfigurations = (

fish.xcodeproj/xcshareddata/xcschemes/Makefile.xcscheme

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Scheme
-   LastUpgradeVersion = "0460"
+   LastUpgradeVersion = "0500"
    version = "1.3">
    <BuildAction
       parallelizeBuildables = "YES"

fish.xcodeproj/xcshareddata/xcschemes/base.xcscheme

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Scheme
-   LastUpgradeVersion = "0460"
+   LastUpgradeVersion = "0500"
    version = "1.3">
    <BuildAction
       parallelizeBuildables = "YES"

fish.xcodeproj/xcshareddata/xcschemes/fish.app.xcscheme

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Scheme
-   LastUpgradeVersion = "0460"
+   LastUpgradeVersion = "0500"
    version = "1.3">
    <BuildAction
       parallelizeBuildables = "YES"

fish.xcodeproj/xcshareddata/xcschemes/fishd.xcscheme

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Scheme
-   LastUpgradeVersion = "0460"
+   LastUpgradeVersion = "0500"
    version = "1.3">
    <BuildAction
       parallelizeBuildables = "YES"

fish.xcodeproj/xcshareddata/xcschemes/install_tree.xcscheme

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Scheme
-   LastUpgradeVersion = "0460"
+   LastUpgradeVersion = "0500"
    version = "1.3">
    <BuildAction
       parallelizeBuildables = "YES"

fishd.cpp

@@ -667,13 +667,14 @@ static void daemonize()
             setup_fork_guards();
 
             /*
-              Make fishd ignore the HUP signal.
+              Make fishd ignore the HUP and PIPE signals.
             */
             struct sigaction act;
             sigemptyset(& act.sa_mask);
             act.sa_flags=0;
             act.sa_handler=SIG_IGN;
             sigaction(SIGHUP, &act, 0);
+            sigaction(SIGPIPE, &act, 0);
 
             /*
               Make fishd save and exit on the TERM signal.

function.cpp

@@ -175,7 +175,7 @@ function_info_t::function_info_t(const function_info_t &data, const wchar_t *fil
 {
 }
 
-void function_add(const function_data_t &data, const parser_t &parser)
+void function_add(const function_data_t &data, const parser_t &parser, int definition_line_offset)
 {
     ASSERT_IS_MAIN_THREAD();
 
@@ -189,13 +189,7 @@ void function_add(const function_data_t &data, const parser_t &parser)
     /* Create and store a new function */
     const wchar_t *filename = reader_current_filename();
 
-    int def_offset = -1;
+    const function_map_t::value_type new_pair(data.name, function_info_t(data, filename, definition_line_offset, is_autoload));
-    if (parser.current_block() != NULL)
-    {
-        def_offset = parser.line_number_of_character_at_offset(parser.current_block()->tok_pos);
-    }
-
-    const function_map_t::value_type new_pair(data.name, function_info_t(data, filename, def_offset, is_autoload));
     loaded_functions.insert(new_pair);
 
     /* Add event handlers */

function.h

@@ -92,8 +92,8 @@ public:
 */
 void function_init();
 
-/** Add a function. */
+/** Add a function. definition_line_offset is the line number of the function's definition within its source file */
-void function_add(const function_data_t &data, const parser_t &parser);
+void function_add(const function_data_t &data, const parser_t &parser, int definition_line_offset = 0);
 
 /** Removes a function from our internal table, returning true if it was found and false if not */
 bool function_remove_ignore_autoload(const wcstring &name);

highlight.h

@@ -30,10 +30,19 @@ enum
     highlight_spec_autosuggestion, //autosuggestion
     highlight_spec_selection,
 
+    // Pager support
+    highlight_spec_pager_prefix,
+    highlight_spec_pager_completion,
+    highlight_spec_pager_description,
+    highlight_spec_pager_progress,
+    highlight_spec_pager_secondary,
+
     HIGHLIGHT_SPEC_PRIMARY_MASK = 0xFF,
 
     /* The following values are modifiers */
     highlight_modifier_valid_path = 0x100,
+    highlight_modifier_force_underline = 0x200,
+    highlight_modifier_sloppy_background = 0x300, //hackish, indicates that we should treat a foreground color as background, per certain historical behavior
 
     /* Very special value */
     highlight_spec_invalid = 0xFFFF
@@ -48,6 +57,7 @@ inline highlight_spec_t highlight_get_primary(highlight_spec_t val)
 
 inline highlight_spec_t highlight_make_background(highlight_spec_t val)
 {
+    assert(val >> 16 == 0); //should have nothing in upper bits, otherwise this is already a background
     return val << 16;
 }
 
@@ -65,7 +75,11 @@ struct file_detection_context_t;
    \param error a list in which a description of each error will be inserted. May be 0, in whcich case no error descriptions will be generated.
 */
 void highlight_shell(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
-void highlight_shell_new_parser(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
+
+/**
+   Perform a non-blocking shell highlighting. The function will not do any I/O that may block. As a result, invalid commands may not be detected, etc.
+*/
+void highlight_shell_no_io(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
 
 /**
    Perform syntax highlighting for the text in buff. Matching quotes and paranthesis are highlighted. The result is
@@ -115,9 +129,5 @@ enum
 typedef unsigned int path_flags_t;
 bool is_potential_path(const wcstring &const_path, const wcstring_list_t &directories, path_flags_t flags, wcstring *out_path = NULL);
 
-/* For testing */
-void highlight_shell_classic(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
-void highlight_shell_new_parser(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
-
 #endif
 

input.cpp

@@ -59,6 +59,7 @@
 #include "output.h"
 #include "intern.h"
 #include <vector>
+#include <algorithm>
 
 #define DEFAULT_TERM L"ansi"
 #define MAX_INPUT_FUNCTION_ARGS 20
@@ -71,12 +72,21 @@ struct input_mapping_t
 {
     wcstring seq; /**< Character sequence which generates this event */
     std::vector<wcstring> commands; /**< commands that should be evaluated by this mapping */
+    
+    /* We wish to preserve the user-specified order. This is just an incrementing value. */
+    unsigned int specification_order;
+    
     wcstring mode; /**< mode in which this command should be evaluated */
     wcstring sets_mode; /** new mode that should be switched to after command evaluation */
     
     input_mapping_t(const wcstring &s, const std::vector<wcstring> &c,
                     const wcstring &m = DEFAULT_BIND_MODE,
-                    const wcstring &sm = DEFAULT_BIND_MODE) : seq(s), commands(c), mode(m), sets_mode(sm) {}
+                    const wcstring &sm = DEFAULT_BIND_MODE) : seq(s), commands(c), mode(m), sets_mode(sm)
+    {
+        static unsigned int s_last_input_mapping_specification_order = 0;
+        specification_order = ++s_last_input_mapping_specification_order;
+        
+    }
 };
 
 /**
@@ -86,7 +96,6 @@ struct terminfo_mapping_t
 {
     const wchar_t *name; /**< Name of key */
     const char *seq; /**< Character sequence generated on keypress. Constant string. */
-
 };
 
 
@@ -109,6 +118,7 @@ static const wchar_t * const name_arr[] =
     L"yank",
     L"yank-pop",
     L"complete",
+    L"complete-and-search",
     L"beginning-of-history",
     L"end-of-history",
     L"backward-kill-line",
@@ -116,7 +126,6 @@ static const wchar_t * const name_arr[] =
     L"kill-word",
     L"backward-kill-word",
     L"backward-kill-path-component",
-    L"dump-functions",
     L"history-token-search-backward",
     L"history-token-search-forward",
     L"self-insert",
@@ -125,9 +134,8 @@ static const wchar_t * const name_arr[] =
     L"upcase-word",
     L"downcase-word",
     L"capitalize-word",
-    L"null",
-    L"eof",
     L"vi-arg-digit",
+    L"vi-delete-to",
     L"execute",
     L"beginning-of-buffer",
     L"end-of-buffer",
@@ -142,9 +150,20 @@ static const wchar_t * const name_arr[] =
     L"kill-selection",
     L"forward-jump",
     L"backward-jump",
-    L"and"
+    L"and",
+    L"cancel"
+};
+
+wcstring describe_char(wint_t c)
+{
+    wchar_t initial_cmd_char = R_BEGINNING_OF_LINE;
+    size_t name_count = sizeof name_arr / sizeof *name_arr;
+    if (c >= initial_cmd_char && c < initial_cmd_char + name_count)
+    {
+        return format_string(L"%02x (%ls)", c, name_arr[c - initial_cmd_char]);
+    }
+    return format_string(L"%02x", c);
 }
-;
 
 /**
    Description of each supported input function
@@ -205,6 +224,7 @@ static const wchar_t code_arr[] =
     R_YANK,
     R_YANK_POP,
     R_COMPLETE,
+    R_COMPLETE_AND_SEARCH,
     R_BEGINNING_OF_HISTORY,
     R_END_OF_HISTORY,
     R_BACKWARD_KILL_LINE,
@@ -212,7 +232,6 @@ static const wchar_t code_arr[] =
     R_KILL_WORD,
     R_BACKWARD_KILL_WORD,
     R_BACKWARD_KILL_PATH_COMPONENT,
-    R_DUMP_FUNCTIONS,
     R_HISTORY_TOKEN_SEARCH_BACKWARD,
     R_HISTORY_TOKEN_SEARCH_FORWARD,
     R_SELF_INSERT,
@@ -221,9 +240,8 @@ static const wchar_t code_arr[] =
     R_UPCASE_WORD,
     R_DOWNCASE_WORD,
     R_CAPITALIZE_WORD,
-    R_NULL,
-    R_EOF,
     R_VI_ARG_DIGIT,
+    R_VI_DELETE_TO,
     R_EXECUTE,
     R_BEGINNING_OF_BUFFER,
     R_END_OF_BUFFER,
@@ -238,9 +256,9 @@ static const wchar_t code_arr[] =
     R_KILL_SELECTION,
     R_FORWARD_JUMP,
     R_BACKWARD_JUMP,
-    R_AND
+    R_AND,
-}
+    R_CANCEL
-;
+};
 
 /** Mappings for the current input mode */
 static std::vector<input_mapping_t> mapping_list;
@@ -327,10 +345,25 @@ wchar_t input_function_get_arg(int index)
     return input_function_args[index];
 }
 
-/**
+/* Helper function to compare the lengths of sequences */
-   Returns the function description for the given function code.
+static bool length_is_greater_than(const input_mapping_t &m1, const input_mapping_t &m2)
-*/
+{
+    return m1.seq.size() > m2.seq.size();
+}
 
+static bool specification_order_is_less_than(const input_mapping_t &m1, const input_mapping_t &m2)
+{
+    return m1.specification_order < m2.specification_order;
+}
+
+/* Inserts an input mapping at the correct position. We sort them in descending order by length, so that we test longer sequences first. */
+static void input_mapping_insert_sorted(const input_mapping_t &new_mapping)
+{
+    std::vector<input_mapping_t>::iterator loc = std::lower_bound(mapping_list.begin(), mapping_list.end(), new_mapping, length_is_greater_than);
+    mapping_list.insert(loc, new_mapping);
+}
+
+/* Adds an input mapping */
 void input_mapping_add(const wchar_t *sequence, const wchar_t **commands, size_t commands_len,
                        const wchar_t *mode, const wchar_t *sets_mode)
 {
@@ -341,6 +374,7 @@ void input_mapping_add(const wchar_t *sequence, const wchar_t **commands, size_t
 
     // debug( 0, L"Add mapping from %ls to %ls in mode %ls", escape(sequence, 1), escape(command, 1 ), mode);
 
+    // remove existing mappings with this sequence
     std::vector<wcstring> commands_vector(commands, commands + commands_len);
 
     for (size_t i=0; i<mapping_list.size(); i++)
@@ -713,11 +747,13 @@ wint_t input_readch()
 
 void input_mapping_get_names(wcstring_list_t &lst)
 {
-    size_t i;
+    // Sort the mappings by the user specification order, so we can return them in the same order that the user specified them in
+    std::vector<input_mapping_t> local_list = mapping_list;
+    std::sort(local_list.begin(), local_list.end(), specification_order_is_less_than);
 
-    for (i=0; i<mapping_list.size(); i++)
+    for (size_t i=0; i<local_list.size(); i++)
     {
-        const input_mapping_t &m = mapping_list.at(i);
+        const input_mapping_t &m = local_list.at(i);
         lst.push_back(wcstring(m.seq));
     }
 

input.h

@@ -18,6 +18,8 @@ inputrc information for key bindings.
 /**
    Key codes for inputrc-style keyboard functions that are passed on
    to the caller of input_read()
+   
+   NOTE: IF YOU MODIFY THIS YOU MUST UPDATE THE name_arr AND code_arr VARIABLES TO MATCH!
 */
 enum
 {
@@ -35,6 +37,7 @@ enum
     R_YANK,
     R_YANK_POP,
     R_COMPLETE,
+    R_COMPLETE_AND_SEARCH,
     R_BEGINNING_OF_HISTORY,
     R_END_OF_HISTORY,
     R_BACKWARD_KILL_LINE,
@@ -42,7 +45,6 @@ enum
     R_KILL_WORD,
     R_BACKWARD_KILL_WORD,
     R_BACKWARD_KILL_PATH_COMPONENT,
-    R_DUMP_FUNCTIONS,
     R_HISTORY_TOKEN_SEARCH_BACKWARD,
     R_HISTORY_TOKEN_SEARCH_FORWARD,
     R_SELF_INSERT,
@@ -67,9 +69,11 @@ enum
     R_KILL_SELECTION,
     R_FORWARD_JUMP,
     R_BACKWARD_JUMP,
-    R_AND
+    R_AND,
-}
+    R_CANCEL
-;
+};
+
+wcstring describe_char(wint_t c);
 
 #define R_MIN R_NULL
 #define R_MAX R_AND

input_common.cpp

@@ -251,7 +251,6 @@ wchar_t input_common_readch(int timed)
                 case 0:
                     return 0;
                 default:
-
                     return res;
             }
         }

io.cpp

@@ -123,14 +123,11 @@ void io_buffer_t::read()
 }
 
 
-io_buffer_t *io_buffer_t::create(bool is_input, int fd)
+io_buffer_t *io_buffer_t::create(int fd)
 {
     bool success = true;
-    if (fd == -1)
+    assert(fd >= 0);
-    {
+    io_buffer_t *buffer_redirect = new io_buffer_t(fd);
-        fd = is_input ? STDIN_FILENO : STDOUT_FILENO;
-    }
-    io_buffer_t *buffer_redirect = new io_buffer_t(fd, is_input);
 
     if (exec_pipe(buffer_redirect->pipe_fd) == -1)
     {

io.h

@@ -131,8 +131,8 @@ private:
     /** buffer to save output in */
     std::vector<char> out_buffer;
 
-    io_buffer_t(int f, bool i):
+    io_buffer_t(int f):
-        io_pipe_t(IO_BUFFER, f, i),
+        io_pipe_t(IO_BUFFER, f, false /* not input */),
         out_buffer()
     {
     }
@@ -172,16 +172,12 @@ public:
 
     /**
        Create a IO_BUFFER type io redirection, complete with a pipe and a
-       vector<char> for output. The default file descriptor used is 1 for
+       vector<char> for output. The default file descriptor used is STDOUT_FILENO
-       output buffering and 0 for input buffering.
+       for buffering
 
-       \param is_input set this parameter to zero if the buffer should be
+       \param fd the fd that will be mapped in the child process, typically STDOUT_FILENO
-       used to buffer the output of a command, or non-zero to buffer the
-       input to a command.
-
-       \param fd when -1, determined from is_input.
     */
-    static io_buffer_t *create(bool is_input, int fd = -1);
+    static io_buffer_t *create(int fd);
 };
 
 class io_chain_t : public std::vector<shared_ptr<io_data_t> >

osx/config.h

@@ -218,7 +218,9 @@
 #if __GNUC__ >= 3
 #define __warn_unused __attribute__ ((warn_unused_result))
 #define __sentinel __attribute__ ((sentinel))
+#define __packed __attribute__ ((packed))
 #else
 #define __warn_unused
 #define __sentinel
+#define __packed
 #endif

output.cpp

@@ -52,7 +52,7 @@
 /**
  Number of color names in the col array
  */
-#define COLORS (sizeof(col)/sizeof(wchar_t *))
+#define FISH_COLORS (sizeof(col)/sizeof(wchar_t *))
 
 static int writeb_internal(char c);
 
@@ -622,7 +622,7 @@ int output_color_code(const wcstring &val, bool is_background)
 
         if (! color_name.empty())
         {
-            for (i=0; i<COLORS; i++)
+            for (i=0; i<FISH_COLORS; i++)
             {
                 if (wcscasecmp(col[i], color_name.c_str()) == 0)
                 {

pager.h

@@ -0,0 +1,170 @@
+/** \file pager.h
+  Pager support
+*/
+
+#include "complete.h"
+#include "screen.h"
+#include "reader.h"
+
+#define PAGER_SELECTION_NONE ((size_t)(-1))
+
+/* Represents rendering from the pager */
+class page_rendering_t
+{
+    public:
+    int term_width;
+    int term_height;
+    size_t rows;
+    size_t cols;
+    size_t row_start;
+    size_t row_end;
+    size_t selected_completion_idx;
+    screen_data_t screen_data;
+    
+    size_t remaining_to_disclose;
+    
+    bool search_field_shown;
+    editable_line_t search_field_line;
+    
+    /* Returns a rendering with invalid data, useful to indicate "no rendering" */
+    page_rendering_t();
+};
+
+/* The space between adjacent completions */
+#define PAGER_SPACER_STRING L"  "
+#define PAGER_SPACER_STRING_WIDTH 2
+
+/* How many rows we will show in the "initial" pager */
+#define PAGER_UNDISCLOSED_MAX_ROWS 4
+
+typedef std::vector<completion_t> completion_list_t;
+page_rendering_t render_completions(const completion_list_t &raw_completions, const wcstring &prefix);
+
+class pager_t
+{
+    int available_term_width;
+    int available_term_height;
+    
+    size_t selected_completion_idx;
+    size_t suggested_row_start;
+    
+    /* Fully disclosed means that we show all completions */
+    bool fully_disclosed;
+    
+    /* Whether we show the search field */
+    bool search_field_shown;
+    
+    /* Returns the index of the completion that should draw selected, using the given number of columns */
+    size_t visual_selected_completion_index(size_t rows, size_t cols) const;
+    
+    /** Data structure describing one or a group of related completions */
+    public:
+    struct comp_t
+    {
+        /** The list of all completin strings this entry applies to */
+        wcstring_list_t comp;
+        
+        /** The description */
+        wcstring desc;
+        
+        /** The representative completion */
+        completion_t representative;
+        
+        /** On-screen width of the completion string */
+        int comp_width;
+        
+        /** On-screen width of the description information */
+        int desc_width;
+        
+        /** Preferred total width */
+        int pref_width;
+        
+        /** Minimum acceptable width */
+        int min_width;
+        
+        comp_t() : comp(), desc(), representative(L""), comp_width(0), desc_width(0), pref_width(0), min_width(0)
+        {
+        }
+    };
+    
+    private:
+    typedef std::vector<comp_t> comp_info_list_t;
+    
+    /* The filtered list of completion infos */
+    comp_info_list_t completion_infos;
+    
+    /* The unfiltered list. Note there's a lot of duplication here. */
+    comp_info_list_t unfiltered_completion_infos;
+    
+    wcstring prefix;
+    
+    void note_selection_changed();
+    
+    bool completion_try_print(size_t cols, const wcstring &prefix, const comp_info_list_t &lst, page_rendering_t *rendering, size_t suggested_start_row) const;
+    
+    void recalc_min_widths(comp_info_list_t * lst) const;
+    void measure_completion_infos(std::vector<comp_t> *infos, const wcstring &prefix) const;
+    
+    bool completion_info_passes_filter(const comp_t &info) const;
+    
+    void completion_print(size_t cols, int *width_per_column, size_t row_start, size_t row_stop, const wcstring &prefix, const comp_info_list_t &lst, page_rendering_t *rendering) const;
+    line_t completion_print_item(const wcstring &prefix, const comp_t *c, size_t row, size_t column, int width, bool secondary, bool selected, page_rendering_t *rendering) const;
+
+    
+    public:
+    
+    /* The text of the search field */
+    editable_line_t search_field_line;
+    
+    /* Sets the set of completions */
+    void set_completions(const completion_list_t &comp);
+    
+    /* Sets the prefix */
+    void set_prefix(const wcstring &pref);
+    
+    /* Sets the terminal width and height */
+    void set_term_size(int w, int h);
+    
+    /* Changes the selected completion in the given direction according to the layout of the given rendering. Returns true if the selection changed. */
+    bool select_next_completion_in_direction(selection_direction_t direction, const page_rendering_t &rendering);
+    
+    /* Returns the currently selected completion for the given rendering */
+    const completion_t *selected_completion(const page_rendering_t &rendering) const;
+    
+    /* Indicates the row and column for the given rendering. Returns -1 if no selection. */
+    size_t get_selected_row(const page_rendering_t &rendering) const;
+    size_t get_selected_column(const page_rendering_t &rendering) const;
+    
+    /* Produces a rendering of the completions, at the given term size */
+    page_rendering_t render() const;
+    
+    /* Updates the rendering if it's stale */
+    void update_rendering(page_rendering_t *rendering) const;
+    
+    /* Indicates if there are no completions, and therefore nothing to render */
+    bool empty() const;
+    
+    /* Clears all completions and the prefix */
+    void clear();
+    
+    /* Updates the completions list per the filter */
+    void refilter_completions();
+    
+    /* Sets whether the search field is shown */
+    void set_search_field_shown(bool flag);
+
+    /* Gets whether the search field shown */
+    bool is_search_field_shown() const;
+    
+    /* Indicates if we are navigating our contents */
+    bool is_navigating_contents() const;
+    
+    /* Become fully disclosed */
+    void set_fully_disclosed(bool flag);
+    
+    /* Position of the cursor */
+    size_t cursor_position() const;
+    
+    /* Constructor */
+    pager_t();
+};

parse_constants.h

@@ -6,10 +6,11 @@
 #ifndef fish_parse_constants_h
 #define fish_parse_constants_h
 
+#include "config.h"
+
 #define PARSE_ASSERT(a) assert(a)
 #define PARSER_DIE() do { fprintf(stderr, "Parser dying!\n"); exit_without_destructors(-1); } while (0)
 
-
 enum parse_token_type_t
 {
     token_type_invalid,
@@ -70,7 +71,7 @@ enum parse_token_type_t
 
     LAST_TOKEN_OR_SYMBOL = parse_token_type_terminate,
     FIRST_PARSE_TOKEN_TYPE = parse_token_type_string
-};
+} __packed;
 
 enum parse_keyword_t
 {
@@ -90,16 +91,18 @@ enum parse_keyword_t
     parse_keyword_not,
     parse_keyword_command,
     parse_keyword_builtin,
+    parse_keyword_exec,
 
-    LAST_KEYWORD = parse_keyword_builtin
+    LAST_KEYWORD = parse_keyword_exec
-};
+} __packed;
 
 /* Statement decorations. This matches the order of productions in decorated_statement */
 enum parse_statement_decoration_t
 {
     parse_statement_decoration_none,
     parse_statement_decoration_command,
-    parse_statement_decoration_builtin
+    parse_statement_decoration_builtin,
+    parse_statement_decoration_exec
 };
 
 /* Parse error code list */
@@ -135,6 +138,31 @@ enum
 };
 typedef unsigned int parser_test_error_bits_t;
 
+struct parse_error_t
+{
+    /** Text of the error */
+    wcstring text;
+
+    /** Code for the error */
+    enum parse_error_code_t code;
+
+    /** Offset and length of the token in the source code that triggered this error */
+    size_t source_start;
+    size_t source_length;
+
+    /** Return a string describing the error, suitable for presentation to the user. If skip_caret is false, the offending line with a caret is printed as well */
+    wcstring describe(const wcstring &src) const;
+    
+    /** Return a string describing the error, suitable for presentation to the user, with the given prefix. If skip_caret is false, the offending line with a caret is printed as well */
+    wcstring describe_with_prefix(const wcstring &src, const wcstring &prefix, bool is_interactive, bool skip_caret) const;
+};
+typedef std::vector<parse_error_t> parse_error_list_t;
+
+/* Special source_start value that means unknown */
+#define SOURCE_LOCATION_UNKNOWN (static_cast<size_t>(-1))
+
+/* Helper function to offset error positions by the given amount. This is used when determining errors in a substring of a larger source buffer. */
+void parse_error_offset_source_start(parse_error_list_t *errors, size_t amt);
 
 /** Maximum number of function calls. */
 #define FISH_MAX_STACK_DEPTH 128
@@ -155,6 +183,12 @@ typedef unsigned int parser_test_error_bits_t;
 /** Error message when encountering an illegal command name */
 #define ILLEGAL_CMD_ERR_MSG _( L"Illegal command name '%ls'")
 
+/** Error message when encountering an unknown builtin name */
+#define UNKNOWN_BUILTIN_ERR_MSG _( L"Unknown builtin '%ls'")
+
+/** Error message when encountering a failed expansion, e.g. for the variable name in for loops */
+#define FAILED_EXPANSION_VARIABLE_NAME_ERR_MSG _( L"Unable to expand variable name '%ls'")
+
 /** Error message when encountering an illegal file descriptor */
 #define ILLEGAL_FD_ERR_MSG _( L"Illegal file descriptor in redirection '%ls'")
 
@@ -173,6 +207,31 @@ typedef unsigned int parser_test_error_bits_t;
 /** Error message for Posix-style assignment: foo=bar */
 #define COMMAND_ASSIGN_ERR_MSG _( L"Unknown command '%ls'. Did you mean 'set %ls %ls'? See the help section on the set command by typing 'help set'.")
 
+/**
+   Error issued on invalid variable name
+*/
+#define COMPLETE_VAR_DESC _( L"The '$' character begins a variable name. The character '%lc', which directly followed a '$', is not allowed as a part of a variable name, and variable names may not be zero characters long. To learn more about variable expansion in fish, type 'help expand-variable'.")
+
+/**
+   Error issued on $?
+*/
+#define COMPLETE_YOU_WANT_STATUS _( L"$? is not a valid variable in fish. If you want the exit status of the last command, try $status.")
+
+/**
+   Error issued on invalid variable name
+*/
+#define COMPLETE_VAR_NULL_DESC _( L"The '$' begins a variable name. It was given at the end of an argument. Variable names may not be zero characters long. To learn more about variable expansion in fish, type 'help expand-variable'.")
+
+/**
+   Error issued on invalid variable name
+*/
+#define COMPLETE_VAR_BRACKET_DESC _( L"Did you mean %ls{$%ls}%ls? The '$' character begins a variable name. A bracket, which directly followed a '$', is not allowed as a part of a variable name, and variable names may not be zero characters long. To learn more about variable expansion in fish, type 'help expand-variable'." )
+
+/**
+   Error issued on invalid variable name
+*/
+#define COMPLETE_VAR_PARAN_DESC _( L"Did you mean (COMMAND)? In fish, the '$' character is only used for accessing variables. To learn more about command substitution in fish, type 'help expand-command-substitution'.")
+
 /**
    While block description
 */

parse_execution.cpp

@@ -27,7 +27,30 @@ static bool specific_statement_type_is_redirectable_block(const parse_node_t &no
 
 }
 
-parse_execution_context_t::parse_execution_context_t(const parse_node_tree_t &t, const wcstring &s, parser_t *p) : tree(t), src(s), parser(p), eval_level(0)
+/* Get the name of a redirectable block, for profiling purposes */
+static wcstring profiling_cmd_name_for_redirectable_block(const parse_node_t &node, const parse_node_tree_t &tree, const wcstring &src)
+{
+    assert(specific_statement_type_is_redirectable_block(node));
+    assert(node.has_source());
+    
+    /* Get the source for the block, and cut it at the next statement terminator. */
+    const size_t src_start = node.source_start;
+    size_t src_len = node.source_length;
+
+    const parse_node_tree_t::parse_node_list_t statement_terminator_nodes = tree.find_nodes(node, parse_token_type_end, 1);
+    if (! statement_terminator_nodes.empty())
+    {
+        const parse_node_t *term = statement_terminator_nodes.at(0);
+        assert(term->source_start >= src_start);
+        src_len = term->source_start - src_start;
+    }
+    
+    wcstring result = wcstring(src, src_start, src_len);
+    result.append(L"...");
+    return result;
+}
+
+parse_execution_context_t::parse_execution_context_t(const parse_node_tree_t &t, const wcstring &s, parser_t *p, int initial_eval_level) : tree(t), src(s), parser(p), eval_level(initial_eval_level), executing_node_idx(NODE_OFFSET_INVALID), cached_lineno_offset(0), cached_lineno_count(0)
 {
 }
 
@@ -49,7 +72,8 @@ node_offset_t parse_execution_context_t::get_offset(const parse_node_t &node) co
     const parse_node_t *addr = &node;
     const parse_node_t *base = &this->tree.at(0);
     assert(addr >= base);
-    node_offset_t offset = addr - base;
+    assert(addr - base < SOURCE_OFFSET_INVALID);
+    node_offset_t offset = static_cast<node_offset_t>(addr - base);
     assert(offset < this->tree.size());
     assert(&tree.at(offset) == &node);
     return offset;
@@ -112,7 +136,7 @@ const parse_node_t *parse_execution_context_t::infinite_recursive_statement_in_j
         /* Ok, this is an undecorated plain statement. Get and expand its command */
         wcstring cmd;
         tree.command_for_plain_statement(plain_statement, src, &cmd);
-        expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES);
+        expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES, NULL);
 
         if (cmd == forbidden_function_name)
         {
@@ -138,10 +162,9 @@ enum process_type_t parse_execution_context_t::process_type_for_command(const pa
     /* Determine the process type, which depends on the statement decoration (command, builtin, etc) */
     enum parse_statement_decoration_t decoration = tree.decoration_for_plain_statement(plain_statement);
 
-    /* Do the "exec hack" */
+    if (decoration == parse_statement_decoration_exec)
-    if (decoration != parse_statement_decoration_command && cmd == L"exec")
     {
-        /* Either 'builtin exec' or just plain 'exec', and definitely not 'command exec'. Note we don't allow overriding exec with a function. */
+        /* Always exec */
         process_type = INTERNAL_EXEC;
     }
     else if (decoration == parse_statement_decoration_command)
@@ -303,9 +326,21 @@ parse_execution_result_t parse_execution_context_t::run_if_statement(const parse
         run_job_list(*job_list_to_execute, ib);
     }
     
+    /* It's possible there's a last-minute cancellation, in which case we should not stomp the exit status (#1297) */
+    if (should_cancel_execution(ib))
+    {
+        result = parse_execution_cancelled;
+    }
+
     /* Done */
     parser->pop_block(ib);
 
+    /* Issue 1061: If we executed, then always report success, instead of letting the exit status of the last command linger */
+    if (result == parse_execution_success)
+    {
+        proc_set_last_status(STATUS_BUILTIN_OK);
+    }
+
     return result;
 }
 
@@ -347,8 +382,9 @@ parse_execution_result_t parse_execution_context_t::run_function_statement(const
     if (result == parse_execution_success)
     {
         const wcstring contents_str = get_source(contents);
+        int definition_line_offset = this->line_offset_of_node_at_offset(this->get_offset(contents));
         wcstring error_str;
-        int err = define_function(*parser, argument_list, contents_str, &error_str);
+        int err = define_function(*parser, argument_list, contents_str, definition_line_offset, &error_str);
         proc_set_last_status(err);
 
         if (! error_str.empty())
@@ -402,13 +438,18 @@ parse_execution_result_t parse_execution_context_t::run_for_statement(const pars
     assert(header.type == symbol_for_header);
     assert(block_contents.type == symbol_job_list);
 
-    /* Get the variable name: `for var_name in ...` */
+    /* Get the variable name: `for var_name in ...`. We expand the variable name. It better result in just one. */
     const parse_node_t &var_name_node = *get_child(header, 1, parse_token_type_string);
-    const wcstring for_var_name = get_source(var_name_node);
+    wcstring for_var_name = get_source(var_name_node);
+    if (! expand_one(for_var_name, 0, NULL))
+    {
+        report_error(var_name_node, FAILED_EXPANSION_VARIABLE_NAME_ERR_MSG, for_var_name.c_str());
+        return parse_execution_errored;
+    }
 
     /* Get the contents to iterate over. */
     const parse_node_t *unmatched_wildcard = NULL;
-    wcstring_list_t argument_list = this->determine_arguments(header, &unmatched_wildcard);
+    wcstring_list_t argument_sequence = this->determine_arguments(header, &unmatched_wildcard);
     if (unmatched_wildcard != NULL)
     {
         return report_unmatched_wildcard_error(*unmatched_wildcard);
@@ -416,15 +457,12 @@ parse_execution_result_t parse_execution_context_t::run_for_statement(const pars
 
     parse_execution_result_t ret = parse_execution_success;
 
-    for_block_t *fb = new for_block_t(for_var_name);
+    for_block_t *fb = new for_block_t();
     parser->push_block(fb);
 
-    /* Note that we store the sequence of values in opposite order */
-    std::reverse(argument_list.begin(), argument_list.end());
-    fb->sequence = argument_list;
-
     /* Now drive the for loop. */
-    while (! fb->sequence.empty())
+    const size_t arg_count = argument_sequence.size();
+    for (size_t i=0; i < arg_count; i++)
     {
         if (should_cancel_execution(fb))
         {
@@ -432,10 +470,8 @@ parse_execution_result_t parse_execution_context_t::run_for_statement(const pars
             break;
         }
 
-        const wcstring &for_variable = fb->variable;
+        const wcstring &val = argument_sequence.at(i);
-        const wcstring &val = fb->sequence.back();
+        env_set(for_var_name, val.c_str(),  ENV_LOCAL);
-        env_set(for_variable, val.c_str(),  ENV_LOCAL);
-        fb->sequence.pop_back();
         fb->loop_status = LOOP_NORMAL;
         fb->skip = 0;
 
@@ -476,16 +512,17 @@ parse_execution_result_t parse_execution_context_t::run_switch_statement(const p
     const parse_node_t &switch_value_node = *get_child(statement, 1, parse_token_type_string);
     const wcstring switch_value = get_source(switch_value_node);
 
-    /* Expand it */
+    /* Expand it. We need to offset any errors by the position of the string */
     std::vector<completion_t> switch_values_expanded;
-    int expand_ret = expand_string(switch_value, switch_values_expanded, EXPAND_NO_DESCRIPTIONS);
+    parse_error_list_t errors;
+    int expand_ret = expand_string(switch_value, switch_values_expanded, EXPAND_NO_DESCRIPTIONS, &errors);
+    parse_error_offset_source_start(&errors, switch_value_node.source_start);
+    
     switch (expand_ret)
     {
         case EXPAND_ERROR:
         {
-            result = report_error(switch_value_node,
+            result = report_errors(errors);
-                                  _(L"Could not expand string '%ls'"),
-                                  switch_value.c_str());
             break;
         }
 
@@ -512,7 +549,7 @@ parse_execution_result_t parse_execution_context_t::run_switch_statement(const p
     }
     const wcstring &switch_value_expanded = switch_values_expanded.at(0).completion;
 
-    switch_block_t *sb = new switch_block_t(switch_value_expanded);
+    switch_block_t *sb = new switch_block_t();
     parser->push_block(sb);
 
     if (result == parse_execution_success)
@@ -581,7 +618,6 @@ parse_execution_result_t parse_execution_context_t::run_while_statement(const pa
 
     /* Push a while block */
     while_block_t *wb = new while_block_t();
-    wb->status = WHILE_TEST_FIRST;
     wb->node_offset = this->get_offset(header);
     parser->push_block(wb);
 
@@ -637,29 +673,43 @@ parse_execution_result_t parse_execution_context_t::run_while_statement(const pa
 }
 
 /* Reports an error. Always returns parse_execution_errored, so you can assign the result to an 'errored' variable */
-parse_execution_result_t parse_execution_context_t::report_error(const parse_node_t &node, const wchar_t *fmt, ...)
+parse_execution_result_t parse_execution_context_t::report_error(const parse_node_t &node, const wchar_t *fmt, ...) const
 {
     if (parser->show_errors)
     {
         /* Create an error */
-        parse_error_t error;
+        parse_error_list_t error_list = parse_error_list_t(1);
-        error.source_start = node.source_start;
+        parse_error_t *error = &error_list.at(0);
-        error.source_length = node.source_length;
+        error->source_start = node.source_start;
-        error.code = parse_error_syntax; //hackish
+        error->source_length = node.source_length;
+        error->code = parse_error_syntax; //hackish
 
         va_list va;
         va_start(va, fmt);
-        error.text = vformat_string(fmt, va);
+        error->text = vformat_string(fmt, va);
         va_end(va);
         
+        this->report_errors(error_list);
+    }
+    return parse_execution_errored;
+}
+
+parse_execution_result_t parse_execution_context_t::report_errors(const parse_error_list_t &error_list) const
+{
+    if (parser->show_errors)
+    {
+        if (error_list.empty())
+        {
+            fprintf(stderr, "Bug: Error reported but no error text found.");
+        }
+    
         /* Get a backtrace */
         wcstring backtrace_and_desc;
-        const parse_error_list_t error_list = parse_error_list_t(1, error);
         parser->get_backtrace(src, error_list, &backtrace_and_desc);
         
+        /* Print it */
         fprintf(stderr, "%ls", backtrace_and_desc.c_str());
     }
-
     return parse_execution_errored;
 }
 
@@ -795,7 +845,7 @@ parse_execution_result_t parse_execution_context_t::populate_plain_process(job_t
     assert(got_cmd);
 
     /* Expand it as a command. Return an error on failure. */
-    bool expanded = expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES);
+    bool expanded = expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES, NULL);
     if (! expanded)
     {
         report_error(statement, ILLEGAL_CMD_ERR_MSG, cmd.c_str());
@@ -813,7 +863,7 @@ parse_execution_result_t parse_execution_context_t::populate_plain_process(job_t
     }
 
     wcstring path_to_external_command;
-    if (process_type == EXTERNAL)
+    if (process_type == EXTERNAL || process_type == INTERNAL_EXEC)
     {
         /* Determine the actual command. This may be an implicit cd. */
         bool has_command = path_get_path(cmd, &path_to_external_command);
@@ -913,14 +963,14 @@ wcstring_list_t parse_execution_context_t::determine_arguments(const parse_node_
 
         /* Expand this string */
         std::vector<completion_t> arg_expanded;
-        int expand_ret = expand_string(arg_str, arg_expanded, EXPAND_NO_DESCRIPTIONS);
+        parse_error_list_t errors;
+        int expand_ret = expand_string(arg_str, arg_expanded, EXPAND_NO_DESCRIPTIONS, &errors);
+        parse_error_offset_source_start(&errors, arg_node.source_start);
         switch (expand_ret)
         {
             case EXPAND_ERROR:
             {
-                this->report_error(arg_node,
+                this->report_errors(errors);
-                                   _(L"Could not expand string '%ls'"),
-                                   arg_str.c_str());
                 break;
             }
 
@@ -982,7 +1032,7 @@ bool parse_execution_context_t::determine_io_chain(const parse_node_t &statement
         enum token_type redirect_type = tree.type_for_redirection(redirect_node, src, &source_fd, &target);
 
         /* PCA: I can't justify this EXPAND_SKIP_VARIABLES flag. It was like this when I got here. */
-        bool target_expanded = expand_one(target, no_exec ? EXPAND_SKIP_VARIABLES : 0);
+        bool target_expanded = expand_one(target, no_exec ? EXPAND_SKIP_VARIABLES : 0, NULL);
         if (! target_expanded || target.empty())
         {
             /* Should improve this error message */
@@ -1051,7 +1101,7 @@ bool parse_execution_context_t::determine_io_chain(const parse_node_t &statement
 
     if (out_chain && ! errored)
     {
-        std::swap(*out_chain, result);
+        out_chain->swap(result);
     }
     return ! errored;
 }
@@ -1264,6 +1314,17 @@ parse_execution_result_t parse_execution_context_t::run_1_job(const parse_node_t
     /* Increment the eval_level for the duration of this command */
     scoped_push<int> saved_eval_level(&eval_level, eval_level + 1);
     
+    /* Save the node index */
+    scoped_push<node_offset_t> saved_node_offset(&executing_node_idx, this->get_offset(job_node));
+
+    /* Profiling support */
+    long long start_time = 0, parse_time = 0, exec_time = 0;
+    profile_item_t *profile_item = this->parser->create_profile_item();
+    if (profile_item != NULL)
+    {
+        start_time = get_time();
+    }
+
     /* When we encounter a block construct (e.g. while loop) in the general case, we create a "block process" that has a pointer to its source. This allows us to handle block-level redirections. However, if there are no redirections, then we can just jump into the block directly, which is significantly faster. */
     if (job_is_simple_block(job_node))
     {
@@ -1273,31 +1334,35 @@ parse_execution_result_t parse_execution_context_t::run_1_job(const parse_node_t
         switch (specific_statement.type)
         {
             case symbol_block_statement:
-                return this->run_block_statement(specific_statement);
+                result = this->run_block_statement(specific_statement);
+                break;
 
             case symbol_if_statement:
-                return this->run_if_statement(specific_statement);
+                result = this->run_if_statement(specific_statement);
+                break;
 
             case symbol_switch_statement:
-                return this->run_switch_statement(specific_statement);
+                result = this->run_switch_statement(specific_statement);
+                break;
 
             default:
                 /* Other types should be impossible due to the specific_statement_type_is_redirectable_block check */
                 PARSER_DIE();
                 break;
         }
+        
+        if (profile_item != NULL)
+        {
+            /* Block-types profile a little weird. They have no 'parse' time, and their command is just the block type */
+            exec_time = get_time();
+            profile_item->level=eval_level;
+            profile_item->parse = 0;
+            profile_item->exec=(int)(exec_time-start_time);
+            profile_item->cmd = profiling_cmd_name_for_redirectable_block(specific_statement, this->tree, this->src);
+            profile_item->skipped = result != parse_execution_success;
         }
         
-    /* Profiling support */
+        return result;
-    long long start_time = 0, parse_time = 0, exec_time = 0;
-    const bool do_profile = profile;
-    profile_item_t *profile_item = NULL;
-    if (do_profile)
-    {
-        profile_item = new profile_item_t();
-        profile_item->skipped = 1;
-        profile_items.push_back(profile_item);
-        start_time = get_time();
     }
 
     job_t *j = new job_t(acquire_job_id(), block_io);
@@ -1330,11 +1395,9 @@ parse_execution_result_t parse_execution_context_t::run_1_job(const parse_node_t
 
 
     /* Store time it took to 'parse' the command */
-    if (do_profile)
+    if (profile_item != NULL)
     {
         parse_time = get_time();
-        profile_item->cmd = j->command();
-        profile_item->skipped=parser->current_block()->skip;
     }
 
     if (populated_job)
@@ -1364,20 +1427,22 @@ parse_execution_result_t parse_execution_context_t::run_1_job(const parse_node_t
         }
     }
 
+    if (profile_item != NULL)
+    {
+        exec_time = get_time();
+        profile_item->level=eval_level;
+        profile_item->parse = (int)(parse_time-start_time);
+        profile_item->exec=(int)(exec_time-parse_time);
+        profile_item->cmd = j ? j->command() : wcstring();
+        profile_item->skipped = ! populated_job || result != parse_execution_success;
+    }
+
     /* If the job was skipped, we pretend it ran anyways */
     if (result == parse_execution_skipped)
     {
         result = parse_execution_success;
     }
 
-    if (do_profile)
-    {
-        exec_time = get_time();
-        profile_item->level=eval_level;
-        profile_item->parse = (int)(parse_time-start_time);
-        profile_item->exec=(int)(exec_time-parse_time);
-        profile_item->skipped = ! populated_job;
-    }
 
     /* Clean up jobs. */
     job_reap(0);
@@ -1474,3 +1539,84 @@ parse_execution_result_t parse_execution_context_t::eval_node_at_offset(node_off
 
     return status;
 }
+
+int parse_execution_context_t::line_offset_of_node_at_offset(node_offset_t requested_index)
+{
+    /* If we're not executing anything, return -1 */
+    if (requested_index == NODE_OFFSET_INVALID)
+    {
+        return -1;
+    }
+
+    /* If for some reason we're executing a node without source, return -1 */
+    const parse_node_t &node = tree.at(requested_index);
+    if (! node.has_source())
+    {
+        return -1;
+    }
+    
+    /* Count the number of newlines, leveraging our cache */
+    const size_t offset = tree.at(requested_index).source_start;
+    assert(offset <= src.size());
+
+    /* Easy hack to handle 0 */
+    if (offset == 0)
+    {
+        return 0;
+    }
+    
+    /* We want to return (one plus) the number of newlines at offsets less than the given offset. cached_lineno_count is the number of newlines at indexes less than cached_lineno_offset. */
+    const wchar_t *str = src.c_str();
+    if (offset > cached_lineno_offset)
+    {
+        size_t i;
+        for (i = cached_lineno_offset; str[i] != L'\0' && i < offset; i++)
+        {
+            /* Add one for every newline we find in the range [cached_lineno_offset, offset) */
+            if (str[i] == L'\n')
+            {
+                cached_lineno_count++;
+            }
+        }
+        cached_lineno_offset = i; //note: i, not offset, in case offset is beyond the length of the string
+    }
+    else if (offset < cached_lineno_offset)
+    {
+        /* Subtract one for every newline we find in the range [offset, cached_lineno_offset) */
+        for (size_t i = offset; i < cached_lineno_offset; i++)
+        {
+            if (str[i] == L'\n')
+            {
+                cached_lineno_count--;
+            }
+        }
+        cached_lineno_offset = offset;
+    }
+    return cached_lineno_count;
+}
+
+int parse_execution_context_t::get_current_line_number()
+{
+    int line_number = -1;
+    int line_offset = this->line_offset_of_node_at_offset(this->executing_node_idx);
+    if (line_offset >= 0)
+    {
+        /* The offset is 0 based; the number is 1 based */
+        line_number = line_offset + 1;
+    }
+    return line_number;
+}
+
+int parse_execution_context_t::get_current_source_offset() const
+{
+    int result = -1;
+    if (executing_node_idx != NODE_OFFSET_INVALID)
+    {
+        const parse_node_t &node = tree.at(executing_node_idx);
+        if (node.has_source())
+        {
+            result = static_cast<int>(node.source_start);
+        }
+    }
+    return result;
+}

parse_execution.h

@@ -12,7 +12,6 @@
 #include "proc.h"
 
 class job_t;
-struct profile_item_t;
 struct block_t;
 
 enum parse_execution_result_t
@@ -40,7 +39,13 @@ private:
     //parse_error_list_t errors;
 
     int eval_level;
-    std::vector<profile_item_t*> profile_items;
+    
+    /* The currently executing node index, used to indicate the line number */
+    node_offset_t executing_node_idx;
+    
+    /* Cached line number information */
+    size_t cached_lineno_offset;
+    int cached_lineno_count;
 
     /* No copying allowed */
     parse_execution_context_t(const parse_execution_context_t&);
@@ -60,7 +65,9 @@ private:
     execution_cancellation_reason_t cancellation_reason(const block_t *block) const;
 
     /* Report an error. Always returns true. */
-    parse_execution_result_t report_error(const parse_node_t &node, const wchar_t *fmt, ...);
+    parse_execution_result_t report_error(const parse_node_t &node, const wchar_t *fmt, ...) const;
+    parse_execution_result_t report_errors(const parse_error_list_t &errors) const;
+    
     /* Wildcard error helper */
     parse_execution_result_t report_unmatched_wildcard_error(const parse_node_t &unmatched_wildcard);
 
@@ -102,8 +109,23 @@ private:
     parse_execution_result_t run_job_list(const parse_node_t &job_list_node, const block_t *associated_block);
     parse_execution_result_t populate_job_from_job_node(job_t *j, const parse_node_t &job_node, const block_t *associated_block);
 
+    /* Returns the line number of the node at the given index, indexed from 0. Not const since it touches cached_lineno_offset */
+    int line_offset_of_node_at_offset(node_offset_t idx);
+
 public:
-    parse_execution_context_t(const parse_node_tree_t &t, const wcstring &s, parser_t *p);
+    parse_execution_context_t(const parse_node_tree_t &t, const wcstring &s, parser_t *p, int initial_eval_level);
+
+    /* Returns the current eval level */
+    int current_eval_level() const { return eval_level; }
+    
+    /* Returns the current line number, indexed from 1. Not const since it touches cached_lineno_offset */
+    int get_current_line_number();
+
+    /* Returns the source offset, or -1 */
+    int get_current_source_offset() const;
+
+    /* Returns the source string */
+    const wcstring &get_source() const { return src; }
 
     /* Start executing at the given node offset. Returns 0 if there was no error, 1 if there was an error */
     parse_execution_result_t eval_node_at_offset(node_offset_t offset, const block_t *associated_block, const io_chain_t &io);

parse_productions.cpp

@@ -126,7 +126,7 @@ RESOLVE(statement)
     if (token1.type == parse_token_type_string)
     {
         // If we are a function, then look for help arguments
-        // Othewrise, if the next token looks like an option (starts with a dash), then parse it as a decorated statement
+        // Otherwise, if the next token looks like an option (starts with a dash), then parse it as a decorated statement
         if (token1.keyword == parse_keyword_function && token2.is_help_argument)
         {
             return 4;
@@ -357,6 +357,7 @@ PRODUCTIONS(decorated_statement) =
     {symbol_plain_statement},
     {KEYWORD(parse_keyword_command), symbol_plain_statement},
     {KEYWORD(parse_keyword_builtin), symbol_plain_statement},
+    {KEYWORD(parse_keyword_exec), symbol_plain_statement}
 };
 RESOLVE(decorated_statement)
 {
@@ -374,6 +375,8 @@ RESOLVE(decorated_statement)
             return 1;
         case parse_keyword_builtin:
             return 2;
+        case parse_keyword_exec:
+            return 3;
     }
 }
 

parse_tree.cpp

@@ -13,8 +13,15 @@ static bool production_is_empty(const production_t *production)
     return (*production)[0] == token_type_invalid;
 }
 
+void swap2(parse_node_tree_t &a, parse_node_tree_t &b)
+{
+    fprintf(stderr, "Swapping!\n");
+    // This uses the base vector implementation
+    a.swap(b);
+}
+
 /** Returns a string description of this parse error */
-wcstring parse_error_t::describe(const wcstring &src, bool skip_caret) const
+wcstring parse_error_t::describe_with_prefix(const wcstring &src, const wcstring &prefix, bool is_interactive, bool skip_caret) const
 {
     wcstring result = text;
     if (! skip_caret && source_start < src.size() && source_start + source_length <= src.size())
@@ -44,7 +51,7 @@ wcstring parse_error_t::describe(const wcstring &src, bool skip_caret) const
         assert(source_start >= line_start);
 
         // Don't include the caret and line if we're interactive this is the first line, because then it's obvious
-        bool skip_caret = (get_is_interactive() && source_start == 0);
+        bool skip_caret = (is_interactive && source_start == 0);
 
         if (! skip_caret)
         {
@@ -53,15 +60,16 @@ wcstring parse_error_t::describe(const wcstring &src, bool skip_caret) const
             {
                 result.push_back(L'\n');
             }
+            result.append(prefix);
             result.append(src, line_start, line_end - line_start);
 
-
             // Append the caret line. The input source may include tabs; for that reason we construct a "caret line" that has tabs in corresponding positions
+            const wcstring line_to_measure = prefix + wcstring(src, line_start, source_start - line_start);
             wcstring caret_space_line;
             caret_space_line.reserve(source_start - line_start);
-            for (size_t i=line_start; i < source_start; i++)
+            for (size_t i=0; i < line_to_measure.size(); i++)
             {
-                wchar_t wc = src.at(i);
+                wchar_t wc = line_to_measure.at(i);
                 if (wc == L'\t')
                 {
                     caret_space_line.push_back(L'\t');
@@ -88,6 +96,11 @@ wcstring parse_error_t::describe(const wcstring &src, bool skip_caret) const
     return result;
 }
 
+wcstring parse_error_t::describe(const wcstring &src) const
+{
+    return this->describe_with_prefix(src, wcstring(), get_is_interactive(), false);
+}
+
 wcstring parse_errors_description(const parse_error_list_t &errors, const wcstring &src, const wchar_t *prefix)
 {
     wcstring target;
@@ -107,6 +120,24 @@ wcstring parse_errors_description(const parse_error_list_t &errors, const wcstri
     return target;
 }
 
+void parse_error_offset_source_start(parse_error_list_t *errors, size_t amt)
+{
+    assert(errors != NULL);
+    if (amt > 0)
+    {
+        size_t i, max = errors->size();
+        for (i=0; i < max; i++)
+        {
+            parse_error_t *error = &errors->at(i);
+            /* preserve the special meaning of -1 as 'unknown' */
+            if (error->source_start != SOURCE_LOCATION_UNKNOWN)
+            {
+                error->source_start += amt;
+            }
+        }
+    }
+}
+
 /** Returns a string description of the given token type */
 wcstring token_type_description(parse_token_type_t type)
 {
@@ -201,45 +232,45 @@ wcstring token_type_description(parse_token_type_t type)
     return format_string(L"Unknown token type %ld", static_cast<long>(type));
 }
 
+#define LONGIFY(x) L ## x
+#define KEYWORD_MAP(x) { parse_keyword_ ## x , LONGIFY(#x) }
+static const struct
+{
+    const parse_keyword_t keyword;
+    const wchar_t * const name;
+}
+keyword_map[] =
+{
+    KEYWORD_MAP(none),
+    KEYWORD_MAP(if),
+    KEYWORD_MAP(else),
+    KEYWORD_MAP(for),
+    KEYWORD_MAP(in),
+    KEYWORD_MAP(while),
+    KEYWORD_MAP(begin),
+    KEYWORD_MAP(function),
+    KEYWORD_MAP(switch),
+    KEYWORD_MAP(case),
+    KEYWORD_MAP(end),
+    KEYWORD_MAP(and),
+    KEYWORD_MAP(or),
+    KEYWORD_MAP(not),
+    KEYWORD_MAP(command),
+    KEYWORD_MAP(builtin),
+    KEYWORD_MAP(exec)
+};
+
 wcstring keyword_description(parse_keyword_t k)
 {
-    switch (k)
+    if (k >= 0 && k <= LAST_KEYWORD)
     {
-        case parse_keyword_none:
+        return keyword_map[k].name;
-            return L"none";
-        case parse_keyword_if:
-            return L"if";
-        case parse_keyword_else:
-            return L"else";
-        case parse_keyword_for:
-            return L"for";
-        case parse_keyword_in:
-            return L"in";
-        case parse_keyword_while:
-            return L"while";
-        case parse_keyword_begin:
-            return L"begin";
-        case parse_keyword_function:
-            return L"function";
-        case parse_keyword_switch:
-            return L"switch";
-        case parse_keyword_case:
-            return L"case";
-        case parse_keyword_end:
-            return L"end";
-        case parse_keyword_and:
-            return L"and";
-        case parse_keyword_or:
-            return L"or";
-        case parse_keyword_not:
-            return L"not";
-        case parse_keyword_command:
-            return L"command";
-        case parse_keyword_builtin:
-            return L"builtin";
     }
+    else
+    {
         return format_string(L"Unknown keyword type %ld", static_cast<long>(k));
     }
+}
 
 static wcstring token_type_user_presentable_description(parse_token_type_t type, parse_keyword_t keyword)
 {
@@ -398,7 +429,7 @@ static void dump_tree_recursive(const parse_node_tree_t &nodes, const wcstring &
 
     result->push_back(L'\n');
     ++*line;
-    for (size_t child_idx = node.child_start; child_idx < node.child_start + node.child_count; child_idx++)
+    for (node_offset_t child_idx = node.child_start; child_idx < node.child_start + node.child_count; child_idx++)
     {
         dump_tree_recursive(nodes, src, child_idx, indent + 1, result, line, inout_first_node_not_dumped);
     }
@@ -481,7 +512,6 @@ class parse_ll_t
     void parse_error(parse_token_t token, parse_error_code_t code, const wchar_t *format, ...);
     void parse_error_failed_production(struct parse_stack_element_t &elem, parse_token_t token);
     void parse_error_unbalancing_token(parse_token_t token);
-    void append_error_callout(wcstring &error_message, parse_token_t token);
 
     void dump_stack(void) const;
 
@@ -523,11 +553,18 @@ class parse_ll_t
         }
 
         // Get the parent index. But we can't get the parent parse node yet, since it may be made invalid by adding children
-        const size_t parent_node_idx = symbol_stack.back().node_idx;
+        const node_offset_t parent_node_idx = symbol_stack.back().node_idx;
 
         // Add the children. Confusingly, we want our nodes to be in forwards order (last token last, so dumps look nice), but the symbols should be reverse order (last token first, so it's lowest on the stack)
-        const size_t child_start = nodes.size();
+        const size_t child_start_big = nodes.size();
-        size_t child_count = 0;
+        assert(child_start_big < NODE_OFFSET_INVALID);
+        node_offset_t child_start = static_cast<node_offset_t>(child_start_big);
+        
+        // To avoid constructing multiple nodes, we push_back a single one that we modify
+        parse_node_t representative_child(token_type_invalid);
+        representative_child.parent = parent_node_idx;
+        
+        node_offset_t child_count = 0;
         for (size_t i=0; i < MAX_SYMBOLS_PER_PRODUCTION; i++)
         {
             production_element_t elem = (*production)[i];
@@ -536,16 +573,12 @@ class parse_ll_t
                 // All done, bail out
                 break;
             }
-            else
+
-            {
+            // Append the parse node.
-                // Generate the parse node.
+            representative_child.type = production_element_type(elem);
-                parse_token_type_t child_type = production_element_type(elem);
+            nodes.push_back(representative_child);
-                parse_node_t child = parse_node_t(child_type);
-                child.parent = parent_node_idx;
-                nodes.push_back(child);
             child_count++;
         }
-        }
 
         // Update the parent
         parse_node_t &parent_node = nodes.at(parent_node_idx);
@@ -560,7 +593,7 @@ class parse_ll_t
         // Replace the top of the stack with new stack elements corresponding to our new nodes. Note that these go in reverse order.
         symbol_stack.pop_back();
         symbol_stack.reserve(symbol_stack.size() + child_count);
-        size_t idx = child_count;
+        node_offset_t idx = child_count;
         while (idx--)
         {
             production_element_t elem = (*production)[idx];
@@ -572,11 +605,11 @@ class parse_ll_t
 public:
 
     /* Constructor */
-    parse_ll_t() : fatal_errored(false), should_generate_error_messages(true)
+    parse_ll_t(enum parse_token_type_t goal) : fatal_errored(false), should_generate_error_messages(true)
     {
         this->symbol_stack.reserve(16);
         this->nodes.reserve(64);
-        this->reset_symbols_and_nodes();
+        this->reset_symbols_and_nodes(goal);
     }
 
     /* Input */
@@ -597,11 +630,11 @@ public:
         this->should_generate_error_messages = flag;
     }
 
-    /* Clear the parse symbol stack (but not the node tree). Add a new job_list_t goal node. This is called from the constructor */
+    /* Clear the parse symbol stack (but not the node tree). Add a node of the given type as the goal node. This is called from the constructor */
-    void reset_symbols(void);
+    void reset_symbols(enum parse_token_type_t goal);
 
-    /* Clear the parse symbol stack and the node tree. Add a new job_list_t goal node. This is called from the constructor. */
+    /* Clear the parse symbol stack and the node tree. Add a node of the given type as the goal node. This is called from the constructor. */
-    void reset_symbols_and_nodes(void);
+    void reset_symbols_and_nodes(enum parse_token_type_t goal);
 
     /* Once parsing is complete, determine the ranges of intermediate nodes */
     void determine_node_ranges();
@@ -646,18 +679,17 @@ void parse_ll_t::dump_stack(void) const
 // Since children always appear after their parents, we can implement this very simply by walking backwards
 void parse_ll_t::determine_node_ranges(void)
 {
-    const size_t source_start_invalid = -1;
     size_t idx = nodes.size();
     while (idx--)
     {
-        parse_node_t *parent = &nodes.at(idx);
+        parse_node_t *parent = &nodes[idx];
 
         // Skip nodes that already have a source range. These are terminal nodes.
-        if (parent->source_start != source_start_invalid)
+        if (parent->source_start != SOURCE_OFFSET_INVALID)
             continue;
 
         // Ok, this node needs a source range. Get all of its children, and then set its range.
-        size_t min_start = source_start_invalid, max_end = 0; //note source_start_invalid is huge
+        source_offset_t min_start = SOURCE_OFFSET_INVALID, max_end = 0; //note SOURCE_OFFSET_INVALID is huge
         for (node_offset_t i=0; i < parent->child_count; i++)
         {
             const parse_node_t &child = nodes.at(parent->child_offset(i));
@@ -668,7 +700,7 @@ void parse_ll_t::determine_node_ranges(void)
             }
         }
 
-        if (min_start != source_start_invalid)
+        if (min_start != SOURCE_OFFSET_INVALID)
         {
             assert(max_end >= min_start);
             parent->source_start = min_start;
@@ -681,13 +713,13 @@ void parse_ll_t::acquire_output(parse_node_tree_t *output, parse_error_list_t *e
 {
     if (output != NULL)
     {
-        std::swap(*output, this->nodes);
+        output->swap(this->nodes);
     }
     this->nodes.clear();
 
     if (errors != NULL)
     {
-        std::swap(*errors, this->errors);
+        errors->swap(this->errors);
     }
     this->errors.clear();
     this->symbol_stack.clear();
@@ -822,22 +854,22 @@ void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token)
     }
 }
 
-void parse_ll_t::reset_symbols(void)
+void parse_ll_t::reset_symbols(enum parse_token_type_t goal)
 {
-    /* Add a new job_list node, and then reset our symbol list to point at it */
+    /* Add a new goal node, and then reset our symbol list to point at it */
-    node_offset_t where = nodes.size();
+    node_offset_t where = static_cast<node_offset_t>(nodes.size());
-    nodes.push_back(parse_node_t(symbol_job_list));
+    nodes.push_back(parse_node_t(goal));
 
     symbol_stack.clear();
-    symbol_stack.push_back(parse_stack_element_t(symbol_job_list, where)); // goal token
+    symbol_stack.push_back(parse_stack_element_t(goal, where)); // goal token
     this->fatal_errored = false;
 }
 
 /* Reset both symbols and nodes */
-void parse_ll_t::reset_symbols_and_nodes(void)
+void parse_ll_t::reset_symbols_and_nodes(enum parse_token_type_t goal)
 {
     nodes.clear();
-    this->reset_symbols();
+    this->reset_symbols(goal);
 }
 
 static bool type_is_terminal_type(parse_token_type_t type)
@@ -1028,35 +1060,11 @@ static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt)
     parse_keyword_t result = parse_keyword_none;
     if (tok == TOK_STRING)
     {
-
+        for (size_t i=0; i < sizeof keyword_map / sizeof *keyword_map; i++)
-        const struct
         {
-            const wchar_t *txt;
+            if (! wcscmp(keyword_map[i].name, tok_txt))
-            parse_keyword_t keyword;
-        } keywords[] =
             {
-            {L"if", parse_keyword_if},
+                result = keyword_map[i].keyword;
-            {L"else", parse_keyword_else},
-            {L"for", parse_keyword_for},
-            {L"in", parse_keyword_in},
-            {L"while", parse_keyword_while},
-            {L"begin", parse_keyword_begin},
-            {L"function", parse_keyword_function},
-            {L"switch", parse_keyword_switch},
-            {L"case", parse_keyword_case},
-            {L"end", parse_keyword_end},
-            {L"and", parse_keyword_and},
-            {L"or", parse_keyword_or},
-            {L"not", parse_keyword_not},
-            {L"command", parse_keyword_command},
-            {L"builtin", parse_keyword_builtin}
-        };
-
-        for (size_t i=0; i < sizeof keywords / sizeof *keywords; i++)
-        {
-            if (! wcscmp(keywords[i].txt, tok_txt))
-            {
-                result = keywords[i].keyword;
                 break;
             }
         }
@@ -1065,10 +1073,10 @@ static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt)
 }
 
 /* Placeholder invalid token */
-static const parse_token_t kInvalidToken = {token_type_invalid, parse_keyword_none, false, false, -1, -1};
+static const parse_token_t kInvalidToken = {token_type_invalid, parse_keyword_none, false, false, SOURCE_OFFSET_INVALID, 0};
 
 /* Terminal token */
-static const parse_token_t kTerminalToken = {parse_token_type_terminate, parse_keyword_none, false, false, -1, -1};
+static const parse_token_t kTerminalToken = {parse_token_type_terminate, parse_keyword_none, false, false, SOURCE_OFFSET_INVALID, 0};
 
 static inline bool is_help_argument(const wchar_t *txt)
 {
@@ -1096,16 +1104,16 @@ static inline parse_token_t next_parse_token(tokenizer_t *tok)
     result.keyword = keyword_for_token(tok_type, tok_txt);
     result.has_dash_prefix = (tok_txt[0] == L'-');
     result.is_help_argument = result.has_dash_prefix && is_help_argument(tok_txt);
-    result.source_start = (size_t)tok_start;
+    result.source_start = (source_offset_t)tok_start;
-    result.source_length = tok_extent;
+    result.source_length = (source_offset_t)tok_extent;
 
     tok_next(tok);
     return result;
 }
 
-bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t parse_flags, parse_node_tree_t *output, parse_error_list_t *errors)
+bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t parse_flags, parse_node_tree_t *output, parse_error_list_t *errors, parse_token_type_t goal)
 {
-    parse_ll_t parser;
+    parse_ll_t parser(goal);
     parser.set_should_generate_error_messages(errors != NULL);
 
     /* Construct the tokenizer */
@@ -1160,7 +1168,7 @@ bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t parse_flags,
                 /* Mark a special error token, and then keep going */
                 const parse_token_t token = {parse_special_type_parse_error, parse_keyword_none, false, false, queue[error_token_idx].source_start, queue[error_token_idx].source_length};
                 parser.accept_tokens(token, kInvalidToken);
-                parser.reset_symbols();
+                parser.reset_symbols(goal);
             }
             else
             {
@@ -1209,7 +1217,7 @@ const parse_node_t *parse_node_tree_t::get_child(const parse_node_t &parent, nod
 
 const parse_node_t &parse_node_tree_t::find_child(const parse_node_t &parent, parse_token_type_t type) const
 {
-    for (size_t i=0; i < parent.child_count; i++)
+    for (node_offset_t i=0; i < parent.child_count; i++)
     {
         const parse_node_t *child = this->get_child(parent, i);
         if (child->type == type)
@@ -1255,7 +1263,7 @@ static void find_nodes_recursive(const parse_node_tree_t &tree, const parse_node
     if (result->size() < max_count)
     {
         if (parent.type == type) result->push_back(&parent);
-        for (size_t i=0; i < parent.child_count; i++)
+        for (node_offset_t i=0; i < parent.child_count; i++)
         {
             const parse_node_t *child = tree.get_child(parent, i);
             assert(child != NULL);
@@ -1486,13 +1494,13 @@ const parse_node_t *parse_node_tree_t::next_node_in_node_list(const parse_node_t
     const parse_node_t *list_cursor = &node_list;
     const parse_node_t *list_entry = NULL;
 
-    /* Loop while we don't have an item but do have a list. Note that not every node in the list may contain an in item that we care about - e.g. job_list contains blank lines as a production */
+    /* Loop while we don't have an item but do have a list. Note that some nodes may contain nothing - e.g. job_list contains blank lines as a production */
     while (list_entry == NULL && list_cursor != NULL)
     {
         const parse_node_t *next_cursor = NULL;
 
         /* Walk through the children */
-        for (size_t i=0; i < list_cursor->child_count; i++)
+        for (node_offset_t i=0; i < list_cursor->child_count; i++)
         {
             const parse_node_t *child = this->get_child(*list_cursor, i);
             if (child->type == entry_type)

parse_tree.h

@@ -18,25 +18,14 @@
 
 class parse_node_t;
 class parse_node_tree_t;
-typedef size_t node_offset_t;
+
+typedef uint32_t node_offset_t;
+
 #define NODE_OFFSET_INVALID (static_cast<node_offset_t>(-1))
 
-struct parse_error_t
+typedef uint32_t source_offset_t;
-{
-    /** Text of the error */
-    wcstring text;
 
-    /** Code for the error */
+#define SOURCE_OFFSET_INVALID (static_cast<source_offset_t>(-1))
-    enum parse_error_code_t code;
-
-    /** Offset and length of the token in the source code that triggered this error */
-    size_t source_start;
-    size_t source_length;
-
-    /** Return a string describing the error, suitable for presentation to the user. If skip_caret is false, the offending line with a caret is printed as well */
-    wcstring describe(const wcstring &src, bool skip_caret = false) const;
-};
-typedef std::vector<parse_error_t> parse_error_list_t;
 
 /* Returns a description of a list of parse errors */
 wcstring parse_errors_description(const parse_error_list_t &errors, const wcstring &src, const wchar_t *prefix = NULL);
@@ -48,8 +37,8 @@ struct parse_token_t
     enum parse_keyword_t keyword; // Any keyword represented by this token
     bool has_dash_prefix; // Hackish: whether the source contains a dash prefix
     bool is_help_argument; // Hackish: whether the source looks like '-h' or '--help'
-    size_t source_start;
+    source_offset_t source_start;
-    size_t source_length;
+    source_offset_t source_length;
 
     wcstring describe() const;
     wcstring user_presentable_description() const;
@@ -80,35 +69,36 @@ wcstring parse_dump_tree(const parse_node_tree_t &tree, const wcstring &src);
 wcstring token_type_description(parse_token_type_t type);
 wcstring keyword_description(parse_keyword_t type);
 
-/** Class for nodes of a parse tree */
+/** Class for nodes of a parse tree. Since there's a lot of these, the size and order of the fields is important. */
 class parse_node_t
 {
 public:
-
-    /* Type of the node */
-    enum parse_token_type_t type;
-
     /* Start in the source code */
-    size_t source_start;
+    source_offset_t source_start;
 
     /* Length of our range in the source code */
-    size_t source_length;
+    source_offset_t source_length;
 
     /* Parent */
     node_offset_t parent;
 
     /* Children */
     node_offset_t child_start;
+    
+    /* Number of children */
     uint8_t child_count;
 
     /* Which production was used */
     uint8_t production_idx;
     
+    /* Type of the node */
+    enum parse_token_type_t type;
+
     /* Description */
     wcstring describe(void) const;
 
     /* Constructor */
-    explicit parse_node_t(parse_token_type_t ty) : type(ty), source_start(-1), source_length(0), parent(NODE_OFFSET_INVALID), child_start(0), child_count(0), production_idx(-1)
+    explicit parse_node_t(parse_token_type_t ty) : source_start(SOURCE_OFFSET_INVALID), source_length(0), parent(NODE_OFFSET_INVALID), child_start(0), child_count(0), production_idx(-1), type(ty)
     {
     }
 
@@ -121,7 +111,7 @@ public:
     /* Indicate if this node has a range of source code associated with it */
     bool has_source() const
     {
-        return source_start != (size_t)(-1);
+        return source_start != SOURCE_OFFSET_INVALID;
     }
 
     /* Gets source for the node, or the empty string if it has no source */
@@ -140,7 +130,6 @@ public:
     }
 };
 
-
 /* The parse tree itself */
 class parse_node_tree_t : public std::vector<parse_node_t>
 {
@@ -197,8 +186,9 @@ public:
     parse_node_list_t specific_statements_for_job(const parse_node_t &job) const;
 };
 
-/* The big entry point. Parse a string! */
+
-bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags, parse_node_tree_t *output, parse_error_list_t *errors);
+/* The big entry point. Parse a string, attempting to produce a tree for the given goal type */
+bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags, parse_node_tree_t *output, parse_error_list_t *errors, parse_token_type_t goal = symbol_job_list);
 
 /* Fish grammar:
 
@@ -221,18 +211,18 @@ bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags, parse
 # A block is a conditional, loop, or begin/end
 
     if_statement = if_clause else_clause end_command arguments_or_redirections_list
-    if_clause = <IF> job STATEMENT_TERMINATOR job_list
+    if_clause = <IF> job <TOK_END> job_list
     else_clause = <empty> |
                  <ELSE> else_continuation
     else_continuation = if_clause else_clause |
-                        STATEMENT_TERMINATOR job_list
+                        <TOK_END> job_list
 
-    switch_statement = SWITCH <TOK_STRING> STATEMENT_TERMINATOR case_item_list end_command arguments_or_redirections_list
+    switch_statement = SWITCH <TOK_STRING> <TOK_END> case_item_list end_command arguments_or_redirections_list
     case_item_list = <empty> |
                     case_item case_item_list |
                     <TOK_END> case_item_list
 
-    case_item = CASE argument_list STATEMENT_TERMINATOR job_list
+    case_item = CASE argument_list <TOK_END> job_list
 
     block_statement = block_header <TOK_END> job_list end_command arguments_or_redirections_list
     block_header = for_header | while_header | function_header | begin_header
@@ -247,9 +237,9 @@ bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags, parse
 
     boolean_statement = AND statement | OR statement | NOT statement
 
-# A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command"
+# A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command" or "exec"
 
-    decorated_statement = plain_statement | COMMAND plain_statement | BUILTIN plain_statement
+    decorated_statement = plain_statement | COMMAND plain_statement | BUILTIN plain_statement | EXEC plain_statement
     plain_statement = <TOK_STRING> arguments_or_redirections_list optional_background
 
     argument_list = <empty> | argument argument_list
@@ -261,8 +251,6 @@ bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags, parse
 
     redirection = <TOK_REDIRECTION> <TOK_STRING>
 
-    terminator = <TOK_END> | <TOK_BACKGROUND>
-
     optional_background = <empty> | <TOK_BACKGROUND>
 
     end_command = END

parse_util.cpp

@@ -40,6 +40,7 @@
 #include "wildcard.h"
 #include "parse_tree.h"
 #include "parser.h"
+#include "builtin.h"
 
 /**
    Error message for improper use of the exec builtin
@@ -144,12 +145,11 @@ size_t parse_util_get_offset(const wcstring &str, int line, long line_offset)
     }
 
     return off + line_offset2;
-
 }
 
-
+static int parse_util_locate_brackets_of_type(const wchar_t *in, wchar_t **begin, wchar_t **end, bool allow_incomplete, wchar_t open_type, wchar_t close_type)
-int parse_util_locate_cmdsubst(const wchar_t *in, wchar_t **begin, wchar_t **end, bool allow_incomplete)
 {
+    /* open_type is typically ( or [, and close type is the corresponding value */
     wchar_t *pos;
     wchar_t prev=0;
     int syntax_error=0;
@@ -177,7 +177,7 @@ int parse_util_locate_cmdsubst(const wchar_t *in, wchar_t **begin, wchar_t **end
             }
             else
             {
-                if (*pos == '(')
+                if (*pos == open_type)
                 {
                     if ((paran_count == 0)&&(paran_begin==0))
                     {
@@ -186,7 +186,7 @@ int parse_util_locate_cmdsubst(const wchar_t *in, wchar_t **begin, wchar_t **end
                     
                     paran_count++;
                 }
-                else if (*pos == ')')
+                else if (*pos == close_type)
                 {
                     
                     paran_count--;
@@ -235,7 +235,19 @@ int parse_util_locate_cmdsubst(const wchar_t *in, wchar_t **begin, wchar_t **end
     return 1;
 }
 
-int parse_util_locate_cmdsubst_range(const wcstring &str, size_t *inout_cursor_offset, wcstring *out_contents, size_t *out_start, size_t *out_end, bool accept_incomplete)
+
+int parse_util_locate_cmdsubst(const wchar_t *in, wchar_t **begin, wchar_t **end, bool accept_incomplete)
+{
+    return parse_util_locate_brackets_of_type(in, begin, end, accept_incomplete, L'(', L')');
+}
+
+int parse_util_locate_slice(const wchar_t *in, wchar_t **begin, wchar_t **end, bool accept_incomplete)
+{
+    return parse_util_locate_brackets_of_type(in, begin, end, accept_incomplete, L'[', L']');
+}
+
+
+static int parse_util_locate_brackets_range(const wcstring &str, size_t *inout_cursor_offset, wcstring *out_contents, size_t *out_start, size_t *out_end, bool accept_incomplete, wchar_t open_type, wchar_t close_type)
 {
     /* Clear the return values */
     out_contents->clear();
@@ -249,21 +261,21 @@ int parse_util_locate_cmdsubst_range(const wcstring &str, size_t *inout_cursor_o
     /* Defer to the wonky version */
     const wchar_t * const buff = str.c_str();
     const wchar_t * const valid_range_start = buff + *inout_cursor_offset, *valid_range_end = buff + str.size();
-    wchar_t *cmdsub_begin = NULL, *cmdsub_end = NULL;
+    wchar_t *bracket_range_begin = NULL, *bracket_range_end = NULL;
-    int ret = parse_util_locate_cmdsubst(valid_range_start, &cmdsub_begin, &cmdsub_end, accept_incomplete);
+    int ret = parse_util_locate_brackets_of_type(valid_range_start, &bracket_range_begin, &bracket_range_end, accept_incomplete, open_type, close_type);
     if (ret > 0)
     {
         /* The command substitutions must not be NULL and must be in the valid pointer range, and the end must be bigger than the beginning */
-        assert(cmdsub_begin != NULL && cmdsub_begin >= valid_range_start && cmdsub_begin <= valid_range_end);
+        assert(bracket_range_begin != NULL && bracket_range_begin >= valid_range_start && bracket_range_begin <= valid_range_end);
-        assert(cmdsub_end != NULL && cmdsub_end > cmdsub_begin && cmdsub_end >= valid_range_start && cmdsub_end <= valid_range_end);
+        assert(bracket_range_end != NULL && bracket_range_end > bracket_range_begin && bracket_range_end >= valid_range_start && bracket_range_end <= valid_range_end);
 
         /* Assign the substring to the out_contents */
-        const wchar_t *interior_begin = cmdsub_begin + 1;
+        const wchar_t *interior_begin = bracket_range_begin + 1;
-        out_contents->assign(interior_begin, cmdsub_end - interior_begin);
+        out_contents->assign(interior_begin, bracket_range_end - interior_begin);
 
         /* Return the start and end */
-        *out_start = cmdsub_begin - buff;
+        *out_start = bracket_range_begin - buff;
-        *out_end = cmdsub_end - buff;
+        *out_end = bracket_range_end - buff;
 
         /* Update the inout_cursor_offset. Note this may cause it to exceed str.size(), though overflow is not likely */
         *inout_cursor_offset = 1 + *out_end;
@@ -271,6 +283,16 @@ int parse_util_locate_cmdsubst_range(const wcstring &str, size_t *inout_cursor_o
     return ret;
 }
 
+int parse_util_locate_cmdsubst_range(const wcstring &str, size_t *inout_cursor_offset, wcstring *out_contents, size_t *out_start, size_t *out_end, bool accept_incomplete)
+{
+    return parse_util_locate_brackets_range(str, inout_cursor_offset, out_contents, out_start, out_end, accept_incomplete, L'(', L')');
+}
+
+int parse_util_locate_slice_range(const wcstring &str, size_t *inout_cursor_offset, wcstring *out_contents, size_t *out_start, size_t *out_end, bool accept_incomplete)
+{
+    return parse_util_locate_brackets_range(str, inout_cursor_offset, out_contents, out_start, out_end, accept_incomplete, L'[', L']');
+}
+
 void parse_util_cmdsubst_extent(const wchar_t *buff, size_t cursor_pos, const wchar_t **a, const wchar_t **b)
 {
     const wchar_t * const cursor = buff + cursor_pos;
@@ -965,6 +987,18 @@ static int parser_is_pipe_forbidden(const wcstring &word)
                     L"continue");
 }
 
+bool parse_util_argument_is_help(const wchar_t *s, int min_match)
+{
+    CHECK(s, 0);
+
+    size_t len = wcslen(s);
+
+    min_match = maxi(min_match, 3);
+
+    return (wcscmp(L"-h", s) == 0) ||
+           (len >= (size_t)min_match && (wcsncmp(L"--help", s, len) == 0));
+}
+
 // Check if the first argument under the given node is --help
 static bool first_argument_is_help(const parse_node_tree_t &node_tree, const parse_node_t &node, const wcstring &src)
 {
@@ -975,16 +1009,216 @@ static bool first_argument_is_help(const parse_node_tree_t &node_tree, const par
         // Check the first argument only
         const parse_node_t &arg = *arg_nodes.at(0);
         const wcstring first_arg_src = arg.get_source(src);
-        is_help = parser_t::is_help(first_arg_src.c_str(), 3);
+        is_help = parse_util_argument_is_help(first_arg_src.c_str(), 3);
     }
     return is_help;
 }
 
+void parse_util_expand_variable_error(const parse_node_t &node, const wcstring &token, size_t token_pos, size_t error_pos, parse_error_list_t *out_errors)
+{
+    size_t stop_pos = token_pos+1;
+
+    switch (token[stop_pos])
+    {
+        case BRACKET_BEGIN:
+        {
+            wchar_t *cpy = wcsdup(token.c_str());
+            *(cpy+token_pos)=0;
+            wchar_t *name = &cpy[stop_pos+1];
+            wchar_t *end = wcschr(name, BRACKET_END);
+            wchar_t *post = NULL;
+            int is_var=0;
+            if (end)
+            {
+                post = end+1;
+                *end = 0;
+
+                if (!wcsvarname(name))
+                {
+                    is_var = 1;
+                }
+            }
+
+            if (is_var)
+            {
+                append_syntax_error(out_errors,
+                                    node,
+                                    COMPLETE_VAR_BRACKET_DESC,
+                                    cpy,
+                                    name,
+                                    post);
+            }
+            else
+            {
+                append_syntax_error(out_errors,
+                                    node,
+                                    COMPLETE_VAR_BRACKET_DESC,
+                                    L"",
+                                    L"VARIABLE",
+                                    L"");
+            }
+            free(cpy);
+
+            break;
+        }
+
+        case INTERNAL_SEPARATOR:
+        {
+            append_syntax_error(out_errors,
+                                node,
+                                COMPLETE_VAR_PARAN_DESC);
+            break;
+        }
+
+        case 0:
+        {
+            append_syntax_error(out_errors,
+                                node,
+                                COMPLETE_VAR_NULL_DESC);
+            break;
+        }
+
+        default:
+        {
+            wchar_t token_stop_char = token[stop_pos];
+            // Unescape (see http://github.com/fish-shell/fish-shell/issues/50)
+            if (token_stop_char == ANY_CHAR)
+                token_stop_char = L'?';
+            else if (token_stop_char == ANY_STRING || token_stop_char == ANY_STRING_RECURSIVE)
+                token_stop_char = L'*';
+
+            append_syntax_error(out_errors,
+                                node,
+                                (token_stop_char == L'?' ? COMPLETE_YOU_WANT_STATUS : COMPLETE_VAR_DESC),
+                                token_stop_char);
+            break;
+        }
+    }
+}
+
+
+/**
+   Test if this argument contains any errors. Detected errors include
+   syntax errors in command substitutions, improperly escaped
+   characters and improper use of the variable expansion operator.
+*/
+parser_test_error_bits_t parse_util_detect_errors_in_argument(const parse_node_t &node, const wcstring &arg_src, parse_error_list_t *out_errors)
+{
+    assert(node.type == symbol_argument);
+
+    int err=0;
+
+    wchar_t *paran_begin, *paran_end;
+    wchar_t *arg_cpy;
+    int do_loop = 1;
+
+    arg_cpy = wcsdup(arg_src.c_str());
+
+    while (do_loop)
+    {
+        switch (parse_util_locate_cmdsubst(arg_cpy,
+                                           &paran_begin,
+                                           &paran_end,
+                                           false))
+        {
+            case -1:
+            {
+                err=1;
+                if (out_errors)
+                {
+                    append_syntax_error(out_errors, node, L"Mismatched parenthesis");
+                }
+                free(arg_cpy);
+                return err;
+            }
+
+            case 0:
+            {
+                do_loop = 0;
+                break;
+            }
+
+            case 1:
+            {
+
+                const wcstring subst(paran_begin + 1, paran_end);
+                wcstring tmp;
+
+                tmp.append(arg_cpy, paran_begin - arg_cpy);
+                tmp.push_back(INTERNAL_SEPARATOR);
+                tmp.append(paran_end+1);
+
+                parse_error_list_t subst_errors;
+                err |= parse_util_detect_errors(subst, &subst_errors);
+
+                /* Our command substitution produced error offsets relative to its source. Tweak the offsets of the errors in the command substitution to account for both its offset within the string, and the offset of the node */
+                size_t error_offset = (paran_begin + 1 - arg_cpy) + node.source_start;
+                parse_error_offset_source_start(&subst_errors, error_offset);
+
+                if (out_errors != NULL)
+                {
+                    out_errors->insert(out_errors->end(), subst_errors.begin(), subst_errors.end());
+                }
+
+                free(arg_cpy);
+                arg_cpy = wcsdup(tmp.c_str());
+
+                break;
+            }
+        }
+    }
+
+    wcstring unesc;
+    if (! unescape_string(arg_cpy, &unesc, UNESCAPE_SPECIAL))
+    {
+        if (out_errors)
+        {
+            append_syntax_error(out_errors, node, L"Invalid token '%ls'", arg_cpy);
+        }
+        return 1;
+    }
+    else
+    {
+        /* Check for invalid variable expansions */
+        const size_t unesc_size = unesc.size();
+        for (size_t idx = 0; idx < unesc_size; idx++)
+        {
+            switch (unesc.at(idx))
+            {
+                case VARIABLE_EXPAND:
+                case VARIABLE_EXPAND_SINGLE:
+                {
+                    wchar_t next_char = (idx + 1 < unesc_size ? unesc.at(idx + 1) : L'\0');
+
+                    if (next_char != VARIABLE_EXPAND &&
+                            next_char != VARIABLE_EXPAND_SINGLE &&
+                            ! wcsvarchr(next_char))
+                    {
+                        err=1;
+                        if (out_errors)
+                        {
+                            parse_util_expand_variable_error(node, unesc, idx, node.source_start, out_errors);
+                        }
+                    }
+
+                    break;
+                }
+            }
+        }
+    }
+
+    free(arg_cpy);
+
+    return err;
+}
+
 parser_test_error_bits_t parse_util_detect_errors(const wcstring &buff_src, parse_error_list_t *out_errors)
 {
     parse_node_tree_t node_tree;
     parse_error_list_t parse_errors;
 
+    parser_test_error_bits_t res = 0;
+
     // Whether we encountered a parse error
     bool errored = false;
 
@@ -1022,6 +1256,7 @@ parser_test_error_bits_t parse_util_detect_errors(const wcstring &buff_src, pars
     // Verify 'or' and 'and' not used inside pipelines
     // Verify pipes via parser_is_pipe_forbidden
     // Verify return only within a function
+    // Verify no variable expansions
 
     if (! errored)
     {
@@ -1044,26 +1279,47 @@ parser_test_error_bits_t parse_util_detect_errors(const wcstring &buff_src, pars
                     errored = append_syntax_error(&parse_errors, node, EXEC_ERR_MSG, is_and ? L"and" : L"or");
                 }
             }
+            else if (node.type == symbol_argument)
+            {
+                const wcstring arg_src = node.get_source(buff_src);
+                res |= parse_util_detect_errors_in_argument(node, arg_src, &parse_errors);
+            }
             else if (node.type == symbol_plain_statement)
             {
+                // In a few places below, we want to know if we are in a pipeline
+                const bool is_in_pipeline = node_tree.statement_is_in_pipeline(node, true /* count first */);
+                
+                // We need to know the decoration
+                const enum parse_statement_decoration_t decoration = node_tree.decoration_for_plain_statement(node);
+                
+                // Check that we don't try to pipe through exec
+                if (is_in_pipeline && decoration == parse_statement_decoration_exec)
+                {
+                    errored = append_syntax_error(&parse_errors, node, EXEC_ERR_MSG, L"exec");
+                }
+                
                 wcstring command;
                 if (node_tree.command_for_plain_statement(node, buff_src, &command))
                 {
                     // Check that we can expand the command
-                    if (! expand_one(command, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES | EXPAND_SKIP_JOBS))
+                    if (! expand_one(command, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES | EXPAND_SKIP_JOBS, NULL))
+                    {
+                        // TODO: leverage the resulting errors
+                        errored = append_syntax_error(&parse_errors, node, ILLEGAL_CMD_ERR_MSG, command.c_str());
+                    }
+
+                    // Check that it doesn't contain a variable
+                    // Note this check is clumsy (it doesn't allow for escaping) but it matches what we do in parse_execution
+                    if (command.find(L'$') != wcstring::npos)
                     {
                         errored = append_syntax_error(&parse_errors, node, ILLEGAL_CMD_ERR_MSG, command.c_str());
                     }
 
                     // Check that pipes are sound
-                    if (! errored && parser_is_pipe_forbidden(command))
+                    if (! errored && parser_is_pipe_forbidden(command) && is_in_pipeline)
-                    {
-                        // forbidden commands cannot be in a pipeline at all
-                        if (node_tree.statement_is_in_pipeline(node, true /* count first */))
                     {
                         errored = append_syntax_error(&parse_errors, node, EXEC_ERR_MSG, command.c_str());
                     }
-                    }
 
                     // Check that we don't return from outside a function
                     // But we allow it if it's 'return --help'
@@ -1128,12 +1384,17 @@ parser_test_error_bits_t parse_util_detect_errors(const wcstring &buff_src, pars
                             errored = append_syntax_error(&parse_errors, node, (command == L"break" ? INVALID_BREAK_ERR_MSG : INVALID_CONTINUE_ERR_MSG));
                         }
                     }
-                }
+                    
-            }
+                    // Check that we don't do an invalid builtin (#1252)
-        }
+                    if (! errored && decoration == parse_statement_decoration_builtin && ! builtin_exists(command))
+                    {
+                        errored = append_syntax_error(&parse_errors, node, UNKNOWN_BUILTIN_ERR_MSG, command.c_str());
                     }
 
-    parser_test_error_bits_t res = 0;
+                }
+            }
+        }
+    }
 
     if (errored)
         res |= PARSER_TEST_ERROR;

parse_util.h

@@ -28,6 +28,12 @@ int parse_util_locate_cmdsubst(const wchar_t *in,
                                wchar_t **end,
                                bool accept_incomplete);
 
+/** Same as parse_util_locate_cmdsubst, but handles square brackets [ ] */
+int parse_util_locate_slice(const wchar_t *in,
+                            wchar_t **begin,
+                            wchar_t **end,
+                            bool accept_incomplete);
+
 /**
    Alternative API. Iterate over command substitutions.
 
@@ -144,6 +150,15 @@ void parse_util_set_argv(const wchar_t * const *argv, const wcstring_list_t &nam
 */
 wchar_t *parse_util_unescape_wildcards(const wchar_t *in);
 
+/**
+   Checks if the specified string is a help option.
+
+   \param s the string to test
+   \param min_match is the minimum number of characters that must match in a long style option, i.e. the longest common prefix between --help and any other option. If less than 3, 3 will be assumed.
+*/
+bool parse_util_argument_is_help(const wchar_t *s, int min_match);
+
+
 /**
    Calculates information on the parameter at the specified index.
 
@@ -165,4 +180,11 @@ std::vector<int> parse_util_compute_indents(const wcstring &src);
 
 parser_test_error_bits_t parse_util_detect_errors(const wcstring &buff_src, parse_error_list_t *out_errors = NULL);
 
+/**
+   Test if this argument contains any errors. Detected errors include syntax errors in command substitutions, improperly escaped characters and improper use of the variable expansion operator.
+
+   This does NOT currently detect unterminated quotes.
+*/
+parser_test_error_bits_t parse_util_detect_errors_in_argument(const parse_node_t &node, const wcstring &arg_src, parse_error_list_t *out_errors = NULL);
+
 #endif

parser.h

@@ -64,8 +64,7 @@ enum block_type_t
     SOURCE, /**< Block created by the . (source) builtin */
     EVENT, /**< Block created on event notifier invocation */
     BREAKPOINT, /**< Breakpoint block */
-}
+};
-;
 
 /**
    block_t represents a block of commands.
@@ -78,22 +77,17 @@ protected:
 
 private:
     const block_type_t block_type; /**< Type of block. */
-    bool made_fake;
 
 public:
     block_type_t type() const
     {
-        return this->made_fake ? FAKE : this->block_type;
+        return this->block_type;
     }
 
-    /** Mark a block as fake; this is used by the return statement. */
+    /** Description of the block, for debugging */
-    void mark_as_fake()
+    wcstring description() const;
-    {
-        this->made_fake = true;
-    }
 
     bool skip; /**< Whether execution of the commands in this block should be skipped */
-    bool had_command; /**< Set to non-zero once a command has been executed in this block */
     int tok_pos; /**< The start index of the block */
 
     node_offset_t node_offset; /* Offset of the node */
@@ -104,7 +98,7 @@ public:
     /** The job that is currently evaluated in the specified block. */
     job_t *job;
 
-    /** Name of file that created this block */
+    /** Name of file that created this block. This string is intern'd. */
     const wchar_t *src_filename;
 
     /** Line number where this block was created */
@@ -122,11 +116,6 @@ public:
 
 struct if_block_t : public block_t
 {
-    bool if_expr_evaluated; // whether we've evaluated the if expression
-    bool is_elseif_entry; // whether we're at the beginning of an ELSEIF branch
-    bool any_branch_taken; // whether the clause of the if statement or any elseif has been found to be true
-    bool else_evaluated; // whether we've encountered a terminal else block
-
     if_block_t();
 };
 
@@ -151,22 +140,17 @@ struct source_block_t : public block_t
 
 struct for_block_t : public block_t
 {
-    wcstring variable; // the variable that will be assigned each value in the sequence
+    for_block_t();
-    wcstring_list_t sequence; // the sequence of values
-    for_block_t(const wcstring &var);
 };
 
 struct while_block_t : public block_t
 {
-    int status;
     while_block_t();
 };
 
 struct switch_block_t : public block_t
 {
-    bool switch_taken;
+    switch_block_t();
-    const wcstring switch_value;
-    switch_block_t(const wcstring &sv);
 };
 
 struct fake_block_t : public block_t
@@ -174,12 +158,6 @@ struct fake_block_t : public block_t
     fake_block_t();
 };
 
-struct function_def_block_t : public block_t
-{
-    function_data_t function_data;
-    function_def_block_t();
-};
-
 struct scope_block_t : public block_t
 {
     scope_block_t(block_type_t type); //must be BEGIN, TOP or SUBST
@@ -201,18 +179,6 @@ enum loop_status
 };
 
 
-/**
-   Possible states for a while block
-*/
-enum while_status
-{
-    WHILE_TEST_FIRST, /**< This is the first command of the first lap of a while loop */
-    WHILE_TEST_AGAIN, /**< This is not the first lap of the while loop, but it is the first command of the loop */
-    WHILE_TESTED, /**< This is not the first command in the loop */
-}
-;
-
-
 /**
    Errors that can be generated by the parser
 */
@@ -247,25 +213,19 @@ enum parser_type_t
 
 struct profile_item_t
 {
-    /**
+    /** Time spent executing the specified command, including parse time for nested blocks. */
-       Time spent executing the specified command, including parse time for nested blocks.
-    */
     int exec;
-    /**
+    
-       Time spent parsing the specified command, including execution time for command substitutions.
+    /** Time spent parsing the specified command, including execution time for command substitutions. */
-    */
     int parse;
-    /**
+    
-       The block level of the specified command. nested blocks and command substitutions both increase the block level.
+    /** The block level of the specified command. nested blocks and command substitutions both increase the block level. */
-    */
     size_t level;
-    /**
+    
-       If the execution of this command was skipped.
+    /** If the execution of this command was skipped. */
-    */
+    bool skipped;
-    int skipped;
+
-    /**
+    /** The command string. */
-       The command string.
-    */
     wcstring cmd;
 };
 
@@ -281,71 +241,40 @@ private:
     /** Whether or not we output errors */
     const bool show_errors;
 
-    /** Last error code */
-    int error_code;
-
-    /** Position of last error */
-    int err_pos;
-
     /** Indication that we should skip all blocks */
     bool cancellation_requested;
     
+    /** Indicates that we are within the process of initializing fish */
+    bool is_within_fish_initialization;
+
     /** Stack of execution contexts. We own these pointers and must delete them */
     std::vector<parse_execution_context_t *> execution_contexts;
 
-    /** Description of last error */
-    wcstring err_buff;
-
-    /** Pointer to the current tokenizer */
-    tokenizer_t *current_tokenizer;
-
-    /** String for representing the current line */
-    wcstring lineinfo;
-
-    /** This is the position of the beginning of the currently parsed command */
-    int current_tokenizer_pos;
-
     /** List of called functions, used to help prevent infinite recursion */
     wcstring_list_t forbidden_function;
 
-    /** String index where the current job started. */
-    int job_start_pos;
-
     /** The jobs associated with this parser */
     job_list_t my_job_list;
 
     /** The list of blocks, allocated with new. It's our responsibility to delete these */
     std::vector<block_t *> block_stack;
 
-    /**
+    /** Gets a description of the block stack, for debugging */
-       Keeps track of how many recursive eval calls have been made. Eval
+    wcstring block_stack_description() const;
-       doesn't call itself directly, recursion happens on blocks and on
+
-       command substitutions.
+    /** List of profile items, allocated with new */
-    */
+    std::vector<profile_item_t *> profile_items;
-    int eval_level;
 
     /* No copying allowed */
     parser_t(const parser_t&);
     parser_t& operator=(const parser_t&);
 
-
-    void parse_job_argument_list(process_t *p, job_t *j, tokenizer_t *tok, std::vector<completion_t>&, bool);
-    int parse_job(process_t *p, job_t *j, tokenizer_t *tok);
-    void skipped_exec(job_t * j);
-    void eval_job(tokenizer_t *tok);
-    int parser_test_argument(const wchar_t *arg, wcstring *out, const wchar_t *prefix, int offset);
-    void print_errors(wcstring &target, const wchar_t *prefix);
-    void print_errors_stderr();
-
     /** Create a job */
     job_t *job_create(const io_chain_t &io);
 
     /** Adds a job to the beginning of the job list. */
     void job_add(job_t *job);
 
-public:
-    std::vector<profile_item_t*> profile_items;
-
     /**
        Returns the name of the currently evaluated function if we are
        currently evaluating a function, null otherwise. This is tested by
@@ -354,6 +283,8 @@ public:
     */
     const wchar_t *is_function() const;
 
+public:
+
     /** Get the "principal" parser, whatever that is */
     static parser_t &principal_parser();
 
@@ -368,9 +299,6 @@ public:
     /** Global event blocks */
     event_blockage_list_t global_event_blocks;
 
-    /** Current block level io redirections  */
-    io_chain_t block_io;
-
     /**
       Evaluate the expressions contained in cmd.
 
@@ -380,8 +308,7 @@ public:
 
       \return 0 on success, 1 otherwise
     */
-    int eval(const wcstring &cmd_str, const io_chain_t &io, enum block_type_t block_type);
+    int eval(const wcstring &cmd, const io_chain_t &io, enum block_type_t block_type);
-    int eval_new_parser(const wcstring &cmd, const io_chain_t &io, enum block_type_t block_type);
 
     /** Evaluates a block node at the given node offset in the topmost execution context */
     int eval_block_node(node_offset_t node_idx, const io_chain_t &io, enum block_type_t block_type);
@@ -389,24 +316,12 @@ public:
     /**
       Evaluate line as a list of parameters, i.e. tokenize it and perform parameter expansion and cmdsubst execution on the tokens.
       The output is inserted into output.
+      Errors are ignored.
 
-      \param line Line to evaluate
+      \param arg_src String to evaluate as an argument list
-      \param output List to insert output to
+      \param output List to insert output into
     */
-    /**
+    void expand_argument_list(const wcstring &arg_src, std::vector<completion_t> &output);
-      \param line Line to evaluate
-      \param output List to insert output to
-    */
-    void eval_args(const wchar_t *line, std::vector<completion_t> &output);
-
-    /**
-       Sets the current evaluation error. This function should only be used by libraries that are called by
-
-       \param ec The new error code
-       \param p The character offset at which the error occured
-       \param str The printf-style error message filter
-    */
-    void error(int ec, size_t p, const wchar_t *str, ...);
 
     /**
        Returns a string describing the current parser pisition in the format 'FILENAME (line LINE_NUMBER): LINE'.
@@ -414,23 +329,11 @@ public:
 
        init.fish (line 127): ls|grep pancake
     */
-    const wchar_t *current_line();
+    wcstring current_line();
 
     /** Returns the current line number */
     int get_lineno() const;
 
-    /** Returns the line number for the character at the given index */
-    int line_number_of_character_at_offset(size_t idx) const;
-
-    /** Returns the current position in the latest string of the tokenizer. */
-    int get_pos() const;
-
-    /** Returns the position where the current job started in the latest string of the tokenizer. */
-    int get_job_pos() const;
-
-    /** Set the current position in the latest string of the tokenizer. */
-    void set_pos(int p);
-
     /** Returns the block at the given index. 0 corresponds to the innermost block. Returns NULL when idx is at or equal to the number of blocks. */
     const block_t *block_at_index(size_t idx) const;
     block_t *block_at_index(size_t idx);
@@ -445,15 +348,15 @@ public:
         return block_stack.size();
     }
 
-    /** Get the string currently parsed */
-    const wchar_t *get_buffer() const;
-
     /** Get the list of jobs */
     job_list_t &job_list()
     {
         return my_job_list;
     }
     
+    /* Hackish. In order to correctly report the origin of code with no associated file, we need to know whether it's run during initialization or not. */
+    void set_is_within_fish_initialization(bool flag);
+
     /** Pushes the block. pop_block will call delete on it. */
     void push_block(block_t *newv);
 
@@ -478,6 +381,9 @@ public:
     /** Returns the job with the given pid */
     job_t *job_get_from_pid(int pid);
     
+    /* Returns a new profile item if profiling is active. The caller should fill it in. The parser_t will clean it up. */
+    profile_item_t *create_profile_item();
+
     /**
        Test if the specified string can be parsed, or if more bytes need
        to be read first. The result will have the PARSER_TEST_ERROR bit
@@ -493,12 +399,9 @@ public:
     void get_backtrace(const wcstring &src, const parse_error_list_t &errors, wcstring *output) const;
 
     /**
-       Test if the specified string can be parsed as an argument list,
+       Detect errors in the specified string when parsed as an argument list. Returns true if an error occurred.
-       e.g. sent to eval_args.  The result has the first bit set if the
-       string contains errors, and the second bit is set if the string
-       contains an unclosed block.
     */
-    int test_args(const wchar_t * buff, wcstring *out, const wchar_t *prefix);
+    bool detect_errors_in_argument_list(const wcstring &arg_list_src, wcstring *out_err, const wchar_t *prefix);
 
     /**
        Tell the parser that the specified function may not be run if not
@@ -506,28 +409,16 @@ public:
        of infinite recursion.
     */
     void forbid_function(const wcstring &function);
+
     /**
        Undo last call to parser_forbid_function().
     */
     void allow_function();
 
     /**
-       Initialize static parser data
+       Output profiling data to the given filename
     */
-    void init();
+    void emit_profiling(const char *path) const;
-
-    /**
-       Destroy static parser data
-    */
-    void destroy();
-
-    /**
-       This function checks if the specified string is a help option.
-
-       \param s the string to test
-       \param min_match is the minimum number of characters that must match in a long style option, i.e. the longest common prefix between --help and any other option. If less than 3, 3 will be assumed.
-    */
-    static int is_help(const wchar_t *s, int min_match);
 
     /**
        Returns the file currently evaluated by the parser. This can be
@@ -540,9 +431,6 @@ public:
        Write a stack trace starting at the specified block to the specified wcstring
     */
     void stack_trace(size_t block_idx, wcstring &buff) const;
-
-    int get_block_type(const wchar_t *cmd) const;
-    const wchar_t *get_block_command(int type) const;
 };
 
 /* Temporary */

reader.h

@@ -22,6 +22,52 @@ class parser_t;
 class completion_t;
 class history_t;
 
+/* Helper class for storing a command line */
+class editable_line_t
+{
+    public:
+    
+    /** The command line */
+    wcstring text;
+    
+    /** The current position of the cursor in the command line */
+    size_t position;
+    
+    const wcstring &get_text() const
+    {
+        return text;
+    }
+    
+    /* Gets the length of the text */
+    size_t size() const
+    {
+        return text.size();
+    }
+    
+    bool empty() const
+    {
+        return text.empty();
+    }
+    
+    void clear()
+    {
+        text.clear();
+        position = 0;
+    }
+    
+    wchar_t at(size_t idx)
+    {
+        return text.at(idx);
+    }
+    
+    editable_line_t() : text(), position(0)
+    {
+    }
+    
+    /* Inserts the string at the cursor position */
+    void insert_string(const wcstring &str);
+};
+
 /**
   Read commands from \c fd until encountering EOF
 */
@@ -120,7 +166,7 @@ size_t reader_get_cursor_pos();
    Get the current selection range in the command line.
    Returns false if there is no active selection, true otherwise.
 */
-bool reader_get_selection(size_t &start, size_t &len);
+bool reader_get_selection(size_t *start, size_t *len);
 
 /**
    Return the value of the interrupted flag, which is set by the sigint
@@ -246,10 +292,18 @@ int reader_shell_test(const wchar_t *b);
 /**
    Test whether the interactive reader is in search mode.
 
-   \return o if not in search mode, 1 if in search mode and -1 if not in interactive mode
+   \return 0 if not in search mode, 1 if in search mode and -1 if not in interactive mode
  */
 int reader_search_mode();
 
+/**
+   Test whether the interactive reader has visible pager contents.
+
+   \return 0 if it has pager contents, 1 if it does not have pager contents, and -1 if not in interactive mode
+ */
+int reader_has_pager_contents();
+
+
 /* Given a command line and an autosuggestion, return the string that gets shown to the user. Exposed for testing purposes only. */
 wcstring combine_command_and_autosuggestion(const wcstring &cmdline, const wcstring &autosuggestion);
 
@@ -259,5 +313,4 @@ bool reader_expand_abbreviation_in_command(const wcstring &cmdline, size_t curso
 /* Apply a completion string. Exposed for testing only. */
 wcstring completion_apply_to_command_line(const wcstring &val_str, complete_flags_t flags, const wcstring &command_line, size_t *inout_cursor_pos, bool append_only);
 
-
 #endif

screen.cpp

@@ -45,6 +45,7 @@ efficient way for transforming that to the desired screen content.
 #include "highlight.h"
 #include "screen.h"
 #include "env.h"
+#include "pager.h"
 
 /** The number of characters to indent new blocks */
 #define INDENT_STEP 4
@@ -1027,7 +1028,7 @@ static void s_update(screen_t *scr, const wchar_t *left_prompt, const wchar_t *r
 
     if (! output.empty())
     {
-        write_loop(1, &output.at(0), output.size());
+        write_loop(STDOUT_FILENO, &output.at(0), output.size());
     }
 
     /* We have now synced our actual screen against our desired screen. Note that this is a big assignment! */
@@ -1237,7 +1238,9 @@ void s_write(screen_t *s,
              const int *indent,
              size_t cursor_pos,
              size_t sel_start_pos,
-             size_t sel_stop_pos)
+             size_t sel_stop_pos,
+             const page_rendering_t &pager,
+             bool cursor_position_is_within_pager)
 {
     screen_data_t::cursor_t cursor_arr;
 
@@ -1306,24 +1309,30 @@ void s_write(screen_t *s,
     {
         int color = colors[i];
 
-        if (i == cursor_pos)
+        if (! cursor_position_is_within_pager && i == cursor_pos)
         {
             color = 0;
-        }
-
-        if (i == cursor_pos)
-        {
             cursor_arr = s->desired.cursor;
         }
 
         s_desired_append_char(s, effective_commandline.at(i), color, indent[i], first_line_prompt_space);
     }
-    if (i == cursor_pos)
+    if (! cursor_position_is_within_pager && i == cursor_pos)
     {
         cursor_arr = s->desired.cursor;
     }
 
     s->desired.cursor = cursor_arr;
+    
+    if (cursor_position_is_within_pager)
+    {
+        s->desired.cursor.x = (int)cursor_pos;
+        s->desired.cursor.y = (int)s->desired.line_count();
+    }
+    
+    /* Append pager_data (none if empty) */
+    s->desired.append_lines(pager.screen_data);
+    
     s_update(s, layout.left_prompt.c_str(), layout.right_prompt.c_str());
     s_save_status(s);
 }
@@ -1429,6 +1438,22 @@ void s_reset(screen_t *s, screen_reset_mode_t mode)
     fstat(2, &s->prev_buff_2);
 }
 
+bool screen_force_clear_to_end()
+{
+    bool result = false;
+    if (clr_eos)
+    {
+        data_buffer_t output;
+        s_write_mbs(&output, clr_eos);
+        if (! output.empty())
+        {
+            write_loop(STDOUT_FILENO, &output.at(0), output.size());
+            result = true;
+        }
+    }
+    return result;
+}
+
 screen_t::screen_t() :
     desired(),
     actual(),

screen.h

@@ -13,8 +13,11 @@
 #define FISH_SCREEN_H
 
 #include <vector>
+#include <sys/stat.h>
 #include "highlight.h"
 
+class page_rendering_t;
+
 /**
    A class representing a single line of a screen.
 */
@@ -40,6 +43,17 @@ struct line_t
         colors.push_back(color);
     }
 
+    void append(const wchar_t *txt, highlight_spec_t color)
+    {
+        for (size_t i=0; txt[i]; i++)
+        {
+            text.push_back(txt[i]);
+            colors.push_back(color);
+        }
+    }
+    
+    
+
     size_t size(void) const
     {
         return text.size();
@@ -55,6 +69,12 @@ struct line_t
         return colors.at(idx);
     }
 
+    void append_line(const line_t &line)
+    {
+        text.insert(text.end(), line.text.begin(), line.text.end());
+        colors.insert(colors.end(), line.colors.begin(), line.colors.end());
+    }
+
 };
 
 /**
@@ -94,6 +114,12 @@ public:
         return line_datas.at(idx);
     }
 
+    line_t &insert_line_at_index(size_t idx)
+    {
+        assert(idx <= line_datas.size());
+        return *line_datas.insert(line_datas.begin() + idx, line_t());
+    }
+
     line_t &line(size_t idx)
     {
         return line_datas.at(idx);
@@ -103,6 +129,16 @@ public:
     {
         return line_datas.size();
     }
+    
+    void append_lines(const screen_data_t &d)
+    {
+        this->line_datas.insert(this->line_datas.end(), d.line_datas.begin(), d.line_datas.end());
+    }
+    
+    bool empty() const
+    {
+        return line_datas.empty();
+    }
 };
 
 /**
@@ -184,6 +220,8 @@ public:
     \param cursor_pos where the cursor is
     \param sel_start_pos where the selections starts (inclusive)
     \param sel_stop_pos where the selections ends (inclusive)
+    \param pager_data any pager data, to append to the screen
+    \param position_is_within_pager whether the position is within the pager line (first line)
 */
 void s_write(screen_t *s,
              const wcstring &left_prompt,
@@ -194,7 +232,9 @@ void s_write(screen_t *s,
              const int *indent,
              size_t cursor_pos,
              size_t sel_start_pos,
-             size_t sel_stop_pos);
+             size_t sel_stop_pos,
+             const page_rendering_t &pager_data,
+             bool position_is_within_pager);
 
 /**
     This function resets the screen buffers internal knowledge about
@@ -232,6 +272,9 @@ enum screen_reset_mode_t
 
 void s_reset(screen_t *s, screen_reset_mode_t mode);
 
+/* Issues an immediate clr_eos, returning if it existed */
+bool screen_force_clear_to_end();
+
 /* Returns the length of an escape code. Exposed for testing purposes only. */
 size_t escape_code_length(const wchar_t *code);
 

share/completions/aura.fish

@@ -0,0 +1 @@
+__fish_complete_aura aura

share/completions/git.fish

@@ -65,6 +65,28 @@ function __fish_git_using_command
   return 1
 end
 
+function __fish_git_stash_using_command
+  set cmd (commandline -opc)
+  if [ (count $cmd) -gt 2 ]
+    if [ $cmd[2] = 'stash' -a $argv[1] = $cmd[3] ]
+      return 0
+    end
+  end
+  return 1
+end
+
+function __fish_git_stash_not_using_subcommand
+  set cmd (commandline -opc)
+  if [ (count $cmd) -gt 2 -a $cmd[2] = 'stash' ]
+    return 1
+  end
+  return 0
+end
+
+function __fish_git_complete_stashes
+   command git stash list --format=format:"%gd:%gs" | sed 's/:/\t/'
+end
+
 # general options
 complete -f -c git -n 'not __fish_git_needs_command' -l help -d 'Display the manual of a git command'
 
@@ -327,16 +349,21 @@ complete -f -c git -n '__fish_contains_opt -s v' -a '(__fish_git_tags)' -d 'Tag'
 
 ### stash
 complete -c git -n '__fish_git_needs_command' -a stash -d 'Stash away changes'
-complete -f -c git -n '__fish_git_using_command stash' -a list -d 'List stashes'
+complete -f -c git -n '__fish_git_using_command stash; and __fish_git_stash_not_using_subcommand' -a list -d 'List stashes'
-complete -f -c git -n '__fish_git_using_command stash' -a show -d 'Show the changes recorded in the stash'
+complete -f -c git -n '__fish_git_using_command stash; and __fish_git_stash_not_using_subcommand' -a show -d 'Show the changes recorded in the stash'
-complete -f -c git -n '__fish_git_using_command stash' -a pop -d 'Apply and remove a single stashed state'
+complete -f -c git -n '__fish_git_using_command stash; and __fish_git_stash_not_using_subcommand' -a pop -d 'Apply and remove a single stashed state'
-complete -f -c git -n '__fish_git_using_command stash' -a apply -d 'Apply a single stashed state'
+complete -f -c git -n '__fish_git_using_command stash; and __fish_git_stash_not_using_subcommand' -a apply -d 'Apply a single stashed state'
-complete -f -c git -n '__fish_git_using_command stash' -a clear -d 'Remove all stashed states'
+complete -f -c git -n '__fish_git_using_command stash; and __fish_git_stash_not_using_subcommand' -a clear -d 'Remove all stashed states'
-complete -f -c git -n '__fish_git_using_command stash' -a drop -d 'Remove a single stashed state from the stash list'
+complete -f -c git -n '__fish_git_using_command stash; and __fish_git_stash_not_using_subcommand' -a drop -d 'Remove a single stashed state from the stash list'
-complete -f -c git -n '__fish_git_using_command stash' -a create -d 'Create a stash'
+complete -f -c git -n '__fish_git_using_command stash; and __fish_git_stash_not_using_subcommand' -a create -d 'Create a stash'
-complete -f -c git -n '__fish_git_using_command stash' -a save -d 'Save a new stash'
+complete -f -c git -n '__fish_git_using_command stash; and __fish_git_stash_not_using_subcommand' -a save -d 'Save a new stash'
-complete -f -c git -n '__fish_git_using_command stash' -a branch -d 'Create a new branch from a stash'
+complete -f -c git -n '__fish_git_using_command stash; and __fish_git_stash_not_using_subcommand' -a branch -d 'Create a new branch from a stash'
-# TODO other options
+
+complete -f -c git -n '__fish_git_stash_using_command apply' -a '(__fish_git_complete_stashes)'
+complete -f -c git -n '__fish_git_stash_using_command branch' -a '(__fish_git_complete_stashes)'
+complete -f -c git -n '__fish_git_stash_using_command drop' -a '(__fish_git_complete_stashes)'
+complete -f -c git -n '__fish_git_stash_using_command pop' -a '(__fish_git_complete_stashes)'
+complete -f -c git -n '__fish_git_stash_using_command show' -a '(__fish_git_complete_stashes)'
 
 ### config
 complete -f -c git -n '__fish_git_needs_command' -a config -d 'Set and read git configuration variables'

share/completions/sshfs.fish

@@ -15,9 +15,9 @@ complete -x -c sshfs -d Hostname -a "
 (__fish_print_users)@
 "
 #
-# Mount Points, for neatness, I am only mounting under ~/mnt/
+# Mount Points
 #
-complete -c sshfs --description "Mount point" -x -a '(find ~/mnt -type d)'
+complete -c sshfs -x -a '(__fish_complete_directories (commandline -ct) "Mount point")'
 #
 # Command options
 #

share/config.fish

@@ -10,6 +10,21 @@
 
 set -g IFS \n\ \t
 
+#
+# Create the default command_not_found handler
+#
+function __fish_default_command_not_found_handler
+    echo "fish: Unknown command '$argv'" >&2
+end
+
+#
+# Hook up the default as the principal command_not_found handler
+# This is likely to be overwritten in __fish_config_interactive
+#
+function __fish_command_not_found_handler --on-event fish_command_not_found
+    __fish_default_command_not_found_handler $argv
+end
+
 #
 # Set default search paths for completions and shellscript functions
 # unless they already exist

share/functions/__fish_complete_aura.fish

@@ -0,0 +1,178 @@
+function __fish_complete_aura -d 'Complete Aura (ARCH/AUR package manager)' --argument-names progname
+    # Completions for aura
+    # Author: Eric Mrak <mail@ericmrak.info>
+    # original for pacman by: Giorgio Lando <patroclo7@gmail.com>
+
+    set -q progname[1]; or set -l progname aura
+
+    set -l listinstalled "(pacman -Q | tr ' ' \t)"
+    set -l listall       "(__fish_print_packages)"
+    set -l listrepos     "(cat /etc/pacman.conf | grep '^\[.\+\]' | sed 's/[]\[]//g')"
+    set -l listgroups    "(pacman -Sg | sed 's/\(.*\)/\1\tPackage group/g')"
+
+    set -l noopt     'commandline | not sgrep -qe "-[a-z]*[ABCDLMOQRSTU]\|--aursync\|--save\|--downgrade\|--viewlog\|--abssync\|--orphans\|--database\|--query\|--sync\|--remove\|--upgrade\|--deptest"'
+    set -l database  'commandline | sgrep -qe "-[a-z]*D\|--database"'
+    set -l query     'commandline | sgrep -qe "-[a-z]*Q\|--query"'
+    set -l remove    'commandline | sgrep -qe "-[a-z]*R\|--remove"'
+    set -l sync      'commandline | sgrep -qe "-[a-z]*S\|--sync"'
+    set -l upgrade   'commandline | sgrep -qe "-[a-z]*U\|--upgrade"'
+    set -l aur       'commandline | sgrep -qe "-[a-z]*A\|--aursync"'
+    set -l abs       'commandline | sgrep -qe "-[a-z]*M\|--abssync"'
+    set -l save      'commandline | sgrep -qe "-[a-z]*B\|--save"'
+    set -l downgrade 'commandline | sgrep -qe "-[a-z]*C\|--downgrade"'
+    set -l orphans   'commandline | sgrep -qe "-[a-z]*O\|--orphans"'
+    set -l logfile   'commandline | sgrep -qe "-[a-z]*L\|--viewlog"'
+    set -l search    'commandline | sgrep -qe "-[a-zA]*s\|--search"'
+
+    # By default fish expands the arguments with the option which is not desired
+    # due to performance reasons.
+    # This will make sure we are expanding an argument and not an option:
+    set -l argument 'not expr -- (commandline --current-token) : "^-.*" > /dev/null'
+
+    # Primary operations
+    complete -c $progname -s A -f -l aursync   -n $noopt -d 'Synchronize AUR packages'
+    complete -c $progname -s B -f -l save      -n $noopt -d 'Save and restore package state'
+    complete -c $progname -s C -f -l downgrade -n $noopt -d 'Package cache actions'
+    complete -c $progname -s D -f -l database  -n $noopt -d 'Modify the package database'
+    complete -c $progname -s L -f -l viewlog   -n $noopt -d 'Pacman log actions'
+    complete -c $progname -s M -f -l abssync   -n $noopt -d 'Build packages from ABS'
+    complete -c $progname -s O -f -l orphans   -n $noopt -d 'Operate on orphan packages'
+    complete -c $progname -s Q -f -l query     -n $noopt -d 'Query the package database'
+    complete -c $progname -s R -f -l remove    -n $noopt -d 'Remove packages from the system'
+    complete -c $progname -s S -f -l sync      -n $noopt -d 'Synchronize packages'
+    complete -c $progname -s T -f -l deptest   -n $noopt -d 'Check dependencies'
+    complete -c $progname -s U -f -l upgrade   -n $noopt -d 'Upgrade or add a local package'
+    complete -c $progname -l auradebug         -d 'Show settings while running'
+    complete -c $progname -l no-pp             -d 'Do not use powerpill'
+    complete -c $progname -l languages         -d 'Show available languages'
+    complete -c $progname -l viewconf          -d 'View pacman.conf'
+    complete -c $progname -s V -f -l version   -d 'Display version and exit'
+    complete -c $progname -s h -f -l help      -d 'Display help'
+
+    # General options
+    complete -c $progname -s b -l dbpath -d 'Alternative database location' -xa '(__fish_complete_directories)'
+    complete -c $progname -s r -l root -d 'Alternative installation root'
+    complete -c $progname -s v -l verbose -d 'Output more status messages'
+    complete -c $progname -l arch      -d 'Alternate architecture'
+    complete -c $progname -l cachedir  -d 'Alternative package cache location'
+    complete -c $progname -l config    -d 'Alternate config file'
+    complete -c $progname -l debug     -d 'Display debug messages'
+    complete -c $progname -l gpgdir    -d 'GPG directory to verify signatures'
+    complete -c $progname -l logfile   -d 'Specify alternative log file'
+    complete -c $progname -l noconfirm -d 'Bypass any question'
+
+    # Transaction options (sync, remove, upgrade)
+    for condition in sync remove upgrade
+        complete -c $progname -n $$condition -s d -l nodeps     -d 'Skip [all] dependency checks'
+        complete -c $progname -n $$condition -l dbonly          -d 'Modify database entry only'
+        complete -c $progname -n $$condition -l noprogressbar   -d 'Do not display progress bar'
+        complete -c $progname -n $$condition -l noscriptlet     -d 'Do not execute install script'
+        complete -c $progname -n $$condition -s p -l print      -d 'Dry run, only print targets'
+        complete -c $progname -n $$condition -l print-format -x -d 'Specify printf-like format'
+    end
+
+    # Database and upgrade options (database, sync, upgrade)
+    for condition in database sync upgrade
+        complete -c $progname -n $$condition      -l asdeps      -d 'Mark PACKAGE as dependency'
+        complete -c $progname -n $$condition      -l asexplicit  -d 'Mark PACKAGE as explicitly installed'
+    end
+
+    # Upgrade options (sync, upgrade)
+    for condition in sync upgrade
+        complete -c $progname -n $$condition -s f -l force       -d 'Bypass file conflict checks'
+        complete -c $progname -n $$condition      -l ignore      -d 'Ignore upgrade of PACKAGE' -xa "$listinstalled"
+        complete -c $progname -n $$condition      -l ignoregroup -d 'Ignore upgrade of GROUP' -xa "$listgroups"
+        complete -c $progname -n $$condition      -l needed      -d 'Do not reinstall up-to-date targets'
+        complete -c $progname -n $$condition      -l recursive   -d 'Recursively reinstall all dependencies'
+    end
+
+    # Query and sync options
+    for condition in query sync
+        complete -c $progname -n $$condition -s g -l groups     -d 'Display all packages in GROUP' -xa "$listgroups"
+        complete -c $progname -n $$condition -s i -l info       -d 'Display information on PACKAGE'
+        complete -c $progname -n $$condition -s q -l quiet      -d 'Show less information'
+        complete -c $progname -n $$condition -s s -l search -r  -d 'Search packages for regexp'
+    end
+
+    for condition in abs aur
+        complete -c $progname -n $$condition -s a -l delmakedeps  -d 'Remove packages only needed during installation'
+        complete -c $progname -n $$condition -s d -l deps         -d 'View package dependencies'
+        complete -c $progname -n $$condition -s i -l info         -d 'View package information'
+        complete -c $progname -n $$condition -s k -l diff         -d 'Show PKGBUILD diffs'
+        complete -c $progname -n $$condition -s p -l pkgbuild     -d 'View the packages\'s PKGBUILD'
+        complete -c $progname -n $$condition -s x -l unsuppress   -d 'Show makepkg output'
+        complete -c $progname -n $$condition -l absdeps           -d 'Build dependencies from ABS'
+    end
+
+    # AUR options
+    complete -c $progname -n $aur -s q -l quiet        -d 'Show less information'
+    complete -c $progname -n $aur -s s -l search -r    -d 'Search AUR by string matching'
+    complete -c $progname -n $aur -s u -l sysupgrade   -d 'Upgrade all installed AUR packages'
+    complete -c $progname -n $aur -s w -l downloadonly -d 'Download the source tarball'
+    complete -c $progname -n $aur -l aurignore -r      -d 'Ignore given comma-separated packages'
+    complete -c $progname -n $aur -l build -r          -d 'Specify a build location'
+    complete -c $progname -n $aur -l builduser -r      -d 'User to build as'
+    complete -c $progname -n $aur -l custom            -d 'Run customizepkg before build'
+    complete -c $progname -n $aur -l devel             -d 'Include -git/-svn/etc packages'
+    complete -c $progname -n $aur -l hotedit           -d 'Prompt for PKGBUILD editing'
+    complete -c $progname -n $aur -l ignorearch        -d 'Ignore architecture checking'
+    complete -c $progname -n "$aur; and $search" -l abc  -d 'Sort alphabetically'
+    complete -c $progname -n "$aur; and $search" -l head -d 'Only show the first 10 results'
+    complete -c $progname -n "$aur; and $search" -l tail -d 'Only show the last 10 results'
+
+    # Backup options
+    complete -c $progname -n $save -s c -l clean -d 'Remove all but the given number of backups'
+    complete -c $progname -n $save -s r -l restore  -d 'Restores a record kept with -B'
+
+    # Downgrade options
+    complete -c $progname -n $downgrade -s b -l backup -d 'Backup to directory'
+    complete -c $progname -n $downgrade -s c -l clean  -d 'Save this many versions'
+    complete -c $progname -n $downgrade -s s -l search -r -d 'Search via regex'
+
+    # Logfile options
+    complete -c $progname -n $logfile -s i -l info   -d 'Show package history'
+    complete -c $progname -n $logfile -s s -l search -r -d 'Search via regex'
+
+    # ABS options
+    complete -c $progname -n $abs -s s -l search -r -d 'Search ABS by regex'
+    complete -c $progname -n $abs -s c -l clean    -d 'Delete local ABS tree'
+    complete -c $progname -n $abs -s y -l refresh  -d 'Download fresh copy of the package list'
+    complete -c $progname -n $abs -s t -l treesync -d 'Sync the given to local ABS tree'
+    complete -c $progname -n $abs -l absdeps       -d 'Download fresh copy of the package list'
+
+    # Orphan options
+    complete -c $progname -n $orphans -s j -l abandon -d 'Uninstall orphan packages'
+
+    # Query options
+    complete -c $progname -n $query -s c -l changelog  -d 'View the change log of PACKAGE'
+    complete -c $progname -n $query -s d -l deps       -d 'List only non-explicit packages (dependencies)'
+    complete -c $progname -n $query -s e -l explicit   -d 'List only explicitly installed packages'
+    complete -c $progname -n $query -s k -l check      -d 'Check if all files owned by PACKAGE are present'
+    complete -c $progname -n $query -s l -l list       -d 'List all files owned by PACKAGE'
+    complete -c $progname -n $query -s m -l foreign    -d 'List all packages not in the database'
+    complete -c $progname -n $query -s o -l owns -r    -d 'Search for the package that owns FILE' -xa ''
+    complete -c $progname -n $query -s p -l file       -d 'Apply the query to a package file, not package' -xa ''
+    complete -c $progname -n $query -s t -l unrequired -d 'List only unrequired packages'
+    complete -c $progname -n $query -s u -l upgrades   -d 'List only out-of-date packages'
+    complete -c $progname -n "$query; and $argument" -xa $listinstalled -d 'Installed package'
+
+    # Remove options
+    complete -c $progname -n $remove -s c -l cascade   -d 'Also remove packages depending on PACKAGE'
+    complete -c $progname -n $remove -s n -l nosave    -d 'Ignore file backup designations'
+    complete -c $progname -n $remove -s s -l recursive -d 'Also remove dependencies of PACKAGE'
+    complete -c $progname -n $remove -s u -l unneeded  -d 'Only remove targets not required by PACKAGE'
+    complete -c $progname -n "$remove; and $argument" -xa $listinstalled -d 'Installed package'
+
+    # Sync options
+    complete -c $progname -n $sync -s c -l clean        -d 'Remove [all] packages from cache'
+    complete -c $progname -n $sync -s l -l list -xa "$listrepos" -d 'List all packages in REPOSITORY'
+    complete -c $progname -n $sync -s u -l sysupgrade   -d 'Upgrade all packages that are out of date'
+    complete -c $progname -n $sync -s w -l downloadonly -d 'Only download the target packages'
+    complete -c $progname -n $sync -s y -l refresh      -d 'Download fresh copy of the package list'
+    complete -c $progname -n "$argument; and $sync" -xa "$listall $listgroups"
+
+    # Upgrade options
+    complete -c $progname -n "$upgrade; and $argument" -xa '(__fish_complete_suffix pkg.tar.xz)' -d 'Package file'
+    complete -c $progname -n "$upgrade; and $argument" -xa '(__fish_complete_suffix pkg.tar.gz)' -d 'Package file'
+end
+

share/functions/__fish_complete_pacman.fish

@@ -6,7 +6,7 @@ function __fish_complete_pacman -d 'Complete pacman (ARCH package manager)' --ar
     set -q progname[1]; or set -l progname pacman
 
     set -l listinstalled "(pacman -Q | tr ' ' \t)"
-    set -l listall       "(pacman -Sl | cut --delim ' ' --fields 2- | tr ' ' \t)"
+    set -l listall       "(__fish_print_packages)"
     set -l listrepos     "(cat /etc/pacman.conf | grep '^\[.\+\]' | sed 's/[]\[]//g')"
     set -l listgroups    "(pacman -Sg | sed 's/\(.*\)/\1\tPackage group/g')"
 
@@ -88,14 +88,14 @@ function __fish_complete_pacman -d 'Complete pacman (ARCH package manager)' --ar
     complete -c $progname -n $query -s p -l file       -d 'Apply the query to a package file, not package' -xa ''
     complete -c $progname -n $query -s t -l unrequired -d 'List only unrequired packages'
     complete -c $progname -n $query -s u -l upgrades   -d 'List only out-of-date packages'
-    complete -c $progname -n "$query; and $argument" -xa $listinstalled -d 'Installed package'
+    complete -c $progname -n "$query; and $argument"   -d 'Installed package' -xa $listinstalled
 
     # Remove options
     complete -c $progname -n $remove -s c -l cascade   -d 'Also remove packages depending on PACKAGE'
     complete -c $progname -n $remove -s n -l nosave    -d 'Ignore file backup designations'
     complete -c $progname -n $remove -s s -l recursive -d 'Also remove dependencies of PACKAGE'
     complete -c $progname -n $remove -s u -l unneeded  -d 'Only remove targets not required by PACKAGE'
-    complete -c $progname -n "$remove; and $argument" -xa $listinstalled -d 'Installed package'
+    complete -c $progname -n "$remove; and $argument" -d 'Installed package' -xa $listinstalled
 
     # Sync options
     complete -c $progname -n $sync -s c -l clean        -d 'Remove [all] packages from cache'

share/functions/__fish_config_interactive.fish

@@ -217,8 +217,8 @@ function __fish_config_interactive -d "Initializations that should be performed
 		end
 	end
 
-	# Load key bindings
+	# Load key bindings. Redirect stderr per #1155
-	__fish_reload_key_bindings
+	__fish_reload_key_bindings ^ /dev/null
 
 	# Repaint screen when window changes size
 	function __fish_winch_handler --on-signal winch
@@ -236,12 +236,29 @@ function __fish_config_interactive -d "Initializations that should be performed
 
 	# The first time a command is not found, look for command-not-found
 	# This is not cheap so we try to avoid doing it during startup
+	# config.fish already installed a handler for noninteractive command-not-found,
+	# so delete it here since we are now interactive
+	functions -e __fish_command_not_found_handler
+
+	# Now install our fancy variant
 	function __fish_command_not_found_setup --on-event fish_command_not_found
 		# Remove fish_command_not_found_setup so we only execute this once
 		functions --erase __fish_command_not_found_setup
 
-		# First check in /usr/lib, this is where modern Ubuntus place this command
+		# First check if we are on OpenSUSE since SUSE's handler has no options
-		if test -f /usr/lib/command-not-found
+		# and expects first argument to be a command and second database
+		# also check if there is command-not-found command.
+		if begin; test -f /etc/SuSE-release; and type -p command-not-found > /dev/null 2> /dev/null; end
+			function __fish_command_not_found_handler --on-event fish_command_not_found
+				/usr/bin/command-not-found $argv
+			end
+		# Check for Fedora's handler
+		else if test -f /usr/libexec/pk-command-not-found
+			function __fish_command_not_found_handler --on-event fish_command_not_found
+				/usr/libexec/pk-command-not-found -- $argv
+			end
+		# Check in /usr/lib, this is where modern Ubuntus place this command
+		else if test -f /usr/lib/command-not-found
 			function __fish_command_not_found_handler --on-event fish_command_not_found
 				/usr/lib/command-not-found -- $argv
 			end
@@ -253,7 +270,7 @@ function __fish_config_interactive -d "Initializations that should be performed
 		# Use standard fish command not found handler otherwise
 		else
 			function __fish_command_not_found_handler --on-event fish_command_not_found
-				echo fish: Unknown command "'$argv'" >&2
+				__fish_default_command_not_found_handler $argv
 			end
 		end
 		__fish_command_not_found_handler $argv

share/functions/__fish_print_make_targets.fish

@@ -1,8 +1,15 @@
 function __fish_print_make_targets
-	set files Makefile makefile GNUmakefile
 	# Some seds (e.g. on Mac OS X), don't support \n in the RHS
 	# Use a literal newline instead
 	# http://sed.sourceforge.net/sedfaq4.html#s4.1
-	sgrep -h -E '^[^#%=$[:space:]][^#%=$]*:([^=]|$)' $files ^/dev/null | cut -d ":" -f 1 | sed -e 's/^ *//;s/ *$//;s/  */\\
+	# The 'rev | cut | rev' trick removes everything after the last colon
+	for file in GNUmakefile Makefile makefile
+		if test -f $file
+			sgrep -h -o -E '^[^#%=$[:space:]][^#%=$]*:([^=]|$)' $file ^/dev/null | rev | cut -d ":" -f 2- | rev | sed -e 's/^ *//;s/ *$//;s/  */\\
 /g' ^/dev/null
+			# On case insensitive filesystems, Makefile and makefile are the same; stop now so we don't double-print 
+			break
 		end
+	end
+end
+

share/functions/__fish_print_packages.fish

@@ -29,6 +29,22 @@ function __fish_print_packages
 		return
 	end
 
+    # Caches for 5 minutes
+	if type -f pacman >/dev/null
+		set cache_file /tmp/.pac-cache.$USER
+		if test -f $cache_file
+			cat $cache_file
+			set age (math (date +%s) - (stat -c '%Y' $cache_file))
+			set max_age 250
+			if test $age -lt $max_age
+				return
+			end
+		end
+
+		# prints: <package name>	Package
+		pacman -Ssq | sed -e 's/$/\t'$package'/' >$cache_file &
+		return
+	end
 
 	# yum is slow, just like rpm, so go to the background
 	if type -f /usr/share/yum-cli/completion-helper.py >/dev/null
@@ -46,7 +62,8 @@ function __fish_print_packages
 		end
 
 		# Remove package version information from output and pipe into cache file
-		/usr/share/yum-cli/completion-helper.py list all -d 0 -C >$cache_file | cut -d '.' -f 1 | sed '1d' | sed '/^\s/d' | sed -e 's/$/'\t$package'/' &
+		/usr/share/yum-cli/completion-helper.py list all -d 0 -C | sed "s/\..*/\t$package/" >$cache_file &
+		return
 	end
 
 	# Rpm is too slow for this job, so we set it up to do completions
@@ -67,7 +84,8 @@ function __fish_print_packages
 		end
 
 		# Remove package version information from output and pipe into cache file
-		rpm -qa | sed -e 's/-[^-]*-[^-]*$//' | sed -e 's/$/'\t$package'/' >$cache_file &
+		rpm -qa |sed -e 's/-[^-]*-[^-]*$/\t'$package'/' >$cache_file &
+		return
 	end
 
 	# This completes the package name from the portage tree.

share/functions/down-or-search.fish

@@ -5,11 +5,18 @@ function down-or-search -d "Depending on cursor position and current mode, eithe
 		return
 	end
 
+	# If we are navigating the pager, then up always navigates
+	if commandline --paging-mode
+		commandline -f down-line
+		return
+	end
+
+
 	# We are not already in search mode.
 	# If we are on the bottom line, start search mode,
 	# otherwise move down
 	set lineno (commandline -L)
-	set line_count (commandline|wc -l)
+	set line_count (commandline | wc -l | tr -d ' ')
 
 	switch $lineno
 		case $line_count

share/functions/fish_default_key_bindings.fish

@@ -117,6 +117,12 @@ function fish_default_key_bindings -d "Default (Emacs-like) key bindings for fis
 	# This will make sure the output of the current command is paged using the less pager when you press Meta-p
 	bind $argv \ep '__fish_paginate'
 	
+	# shift-tab does a tab complete followed by a search
+	bind --key btab complete-and-search
+
+	# escape cancels stuff	
+	bind \e cancel
+
 	# term-specific special bindings
 	switch "$TERM"
 		case 'rxvt*'
@@ -125,3 +131,4 @@ function fish_default_key_bindings -d "Default (Emacs-like) key bindings for fis
 			bind $argv \eOd backward-word
 	end
 end
+

share/functions/isatty.fish

@@ -1,28 +1,27 @@
+function isatty -d "Test if a file or file descriptor is a tty."
 
-function isatty -d "Tests if a file descriptor is a tty"
+# Use `command test` because `builtin test` doesn't open the regular fd's.
-	set -l fd 0
-	if count $argv >/dev/null
-		switch $argv[1]
 
-			case -h --h --he --hel --help
+  switch "$argv"
+
+    case '-h*' '--h*'
       __fish_print_help isatty
-				return 0
 
-			case stdin
+    case ''
-				set fd 0
+      command test -c /dev/stdin
-
-			case stdout
-				set fd 1
-
-			case stderr
-				set fd 2
 
     case '*'
-				set fd $argv[1]
+      if test -e "$argv" # The eval here is needed for symlinks. Unsure why.
+        command test -c "$argv"; and eval tty 0>"$argv" >/dev/null
 
+      else if test -e /dev/"$argv"
+         command test -c /dev/"$argv"; and tty 0>/dev/"$argv" >/dev/null
+
+      else if test -e /dev/fd/"$argv"
+         command test -c /dev/fd/"$argv"; and tty 0>/dev/fd/"$argv" >/dev/null
+
+      else
+         return 1
     end
   end
-
-	eval "tty 0>&$fd >/dev/null"
-
 end

share/functions/math.fish

@@ -8,6 +8,7 @@ function math --description "Perform math calculations in bc"
 		end
 
 		set -l out (echo $argv|env BC_LINE_LENGTH=0 bc)
+                test -z "$out"; and return 1
 		echo $out
 		switch $out
 			case 0

share/functions/seq.fish

@@ -29,14 +29,14 @@ if begin ; not test -x /usr/bin/seq ; and not type -f seq > /dev/null; end
 		
 			case '*'
 				printf (_ "%s: Expected 1, 2 or 3 arguments, got %d\n") seq (count $argv)
-				exit 1
+				return 1
 		
 		end
 		
 		for i in $from $step $to
 			if not echo $i | grep -E '^-?[0-9]*([0-9]*|\.[0-9]+)$' >/dev/null
 				printf (_ "%s: '%s' is not a number\n") seq $i
-				exit 1
+				return 1
 			end
 		end
 		

share/functions/up-or-search.fish

@@ -5,6 +5,12 @@ function up-or-search -d "Depending on cursor position and current mode, either
 		return
 	end
 
+	# If we are navigating the pager, then up always navigates
+	if commandline --paging-mode
+		commandline -f up-line
+		return
+	end
+
 	# We are not already in search mode.
 	# If we are on the top line, start search mode,
 	# otherwise move up

share/functions/vared.fish

@@ -1,4 +1,3 @@
-
 #
 # This is a neat function, stolen from zsh. It allows you to edit the
 # value of a variable interactively.
@@ -39,11 +38,11 @@ function vared --description "Edit variable value"
 					end
 				else
 
-                                        printf (_ '%s: %s is an array variable. Use %svared%s %s[n] to edit the n:th element of %s\n') vared $argv (set_color $fish_color_command) (set_color $fish_color_normal) $argv $argv
+                                        printf (_ '%s: %s is an array variable. Use %svared%s %s[n] to edit the n:th element of %s\n') vared $argv (set_color $fish_color_command; echo) (set_color $fish_color_normal; echo) $argv $argv
 				end
 		end
 	else
-		printf (_ '%s: Expected exactly one argument, got %s.\n\nSynopsis:\n\t%svared%s VARIABLE\n') vared (count $argv) (set_color $fish_color_command) (set_color $fish_color_normal)
+		printf (_ '%s: Expected exactly one argument, got %s.\n\nSynopsis:\n\t%svared%s VARIABLE\n') vared (count $argv) (set_color $fish_color_command; echo) (set_color $fish_color_normal; echo)
 	end
 end
 

share/tools/web_config/fishconfig.css

@@ -1,5 +1,5 @@
 body {
-    background-color: #292939;
+    background-color: #292929;
     font-family: Courier, "Courier New", monospace;
     color: white;
 }
@@ -20,7 +20,7 @@ body {
 #tab_parent {
     display: table;
     width: 100%;
-    height: 50px;;
+    height: 50px;
 }
 
 .tab {
@@ -269,25 +269,26 @@ body {
 }
 
 .colorpicker_term256_row {
+	padding: 0;
 }
 
 .colorpicker_term256_cell {
 	width: 18px;
 	height: 18px;
 	border: solid black 1px;
+	padding: 0;
 }
 
 .colorpicker_term256_selection_indicator {
-	width: 19px;
+	width: 18px;
 	height: 16px;
-	margin: -2px;
+	margin: -4px;
-	border: solid white 3px;
+	border: solid white 4px;
 	position: relative;
 	z-index: 2;
 }
 
 .colorpicker_cell_selected {
-	 border: dashed white 3px;
 	 width: 12px;
 	 height: 12px;
 }
@@ -302,6 +303,18 @@ body {
 	position: relative; /* so that our absolutely positioned elements work */
 }
 
+.cs_clickable {
+	border: dotted 1px #777;
+	padding: 4px;
+	margin: -5px;
+}
+
+.cs_editing {
+	border: solid 3px #3399FF;
+	padding: 4px;
+	margin: -7px;
+}
+
 .colorpicker_text_sample_tight {
 	font-size: 10pt;
 	line-height: 1.2em;
@@ -320,14 +333,22 @@ body {
 }
 
 .color_picker_background_cells div {
-	width: 14px;
+	width: 24px;
-	height: 14px;
+	height: 24px;
 	border-style: solid;
 	border-color: #777;
 	border-width: 0 0 1px 1px;  /* top right bottom left */
 	float: left;
 }
 
+.color_picker_background_cells span {
+	float: left;
+	font-size: 12pt;
+	padding-top: 2px;
+	padding-right: 8px;
+	cursor: pointer;
+}
+
 .color_scheme_choice_label {
 	margin-left: 10px;
 	margin-bottom: 3px;
@@ -336,9 +357,17 @@ body {
 	white-space: normal;
 }
 
+.color_scheme_choices_scrollview {
+	 border-top: 1px solid #333;
+	 padding-top: 5px;
+	 overflow: scroll;
+	 max-height: 18em; /* about two and a half boxes */
+}
+
 .color_scheme_choices_list {
 	overflow-y: hidden; /* makes our height account for floats */
 	padding:  0 10px 15px 10px;  /* top right bottom left */
+	bottom: 0px;
 }
 
 .color_scheme_choice_container {
@@ -399,11 +428,11 @@ img.delete_icon {
 	position: relative; /* so that our absolutely positioned elements work */
 }
 
-.save_button, .prompt_save_button {
+.save_button, .prompt_save_button, .colors_close_button, .customize_theme_button {
 	border-radius: 5px;
 	border: solid rgba(71,71,71,0.5) 1px;
 	padding: 5px 8px;
-	font-size: 10pt;
+	font-size: 13pt;
 	display: inline-block;
 	margin-top: 12px;
 	background-color: rgba(128,128,128,0.2);
@@ -411,6 +440,14 @@ img.delete_icon {
 	cursor: pointer;
 }
 
+.save_button:hover, .customize_theme_button:hover {
+	border-color: rgba(71,71,71,0.9);
+}
+
+.button_highlight {
+	background-color: rgba(128,128,128,0.6)
+}
+
 .prompt_save_button {
 	background-color: #333;
 	border: solid #525252 1px;

share/tools/web_config/index.html

@@ -15,7 +15,7 @@
 <body>
 
 <div id="ancestor">
-    <span style="font-size: 16pt; color: #8888FF">fish</span><p id="global_error" class="error_msg" error-message></p>
+    <span style="font-size: 16pt; color: #CCC">fish</span><p id="global_error" class="error_msg" error-message></p>
     <div id="parent">
         <div id="tab_parent" ng-controller="main">
             <div ng-class="{'tab': true, 'selected_tab': currentTab =='colors'}" id="tab_colors" ng-click="changeView('colors')">colors</div>

share/tools/web_config/js/colorutils.js

@@ -243,6 +243,17 @@ term_256_colors = [ //247
 "ffffff",
 ]
 
+/* Given a color setting name like 'autosuggestion', return the user visible name we present */
+function user_visible_title_for_setting_name(name) {
+	if (! name) return '';
+	switch (name) {
+		case 'param': return 'parameters';
+		case 'escape': return 'escape sequences';
+		case 'end': return 'statement terminators';
+		default: return name + 's';
+	}
+}
+
 /* Returns array of values from a dictionary (or any object) */
 function dict_values(dict) {
     var result = [];
@@ -265,6 +276,11 @@ function get_colors_as_nested_array(colors, items_per_row) {
 
 /* Given an RGB color as a hex string, like FF0033, convert to HSL, apply the function to adjust its lightness, then return the new color as an RGB string */
 function adjust_lightness(color_str, func) {
+	/* Strip off hash prefix */
+	if (color_str[0] == '#') {
+		color_str = color_str.substring(1);
+	}
+	
     /* Hack to handle for example F00 */
     if (color_str.length == 3) {
         color_str = color_str[0] + color_str[0] + color_str[1] + color_str[1] + color_str[2] + color_str[2]

share/tools/web_config/js/controllers.js

@@ -12,23 +12,27 @@ controllers.controller("main", function($scope, $location) {
 
 controllers.controller("colorsController", function($scope, $http) {
     $scope.changeSelectedColorScheme= function(newScheme) {
-        $scope.selectedColorScheme = newScheme;
+        $scope.selectedColorScheme = angular.copy(newScheme);
         if ($scope.selectedColorScheme.preferred_background) {
             $scope.terminalBackgroundColor = $scope.selectedColorScheme.preferred_background;
         }
-        $scope.selectedColorSetting = 'command';
+        $scope.selectedColorSetting = false;
+		$scope.customizationActive = false;
+		$scope.csEditingType = false;
         $scope.colorArraysArray = $scope.getColorArraysArray();
         //TODO: Save button should be shown only when colors are changed
         $scope.showSaveButton = true;
+        
+        $scope.noteThemeChanged();
     }
 
     $scope.changeTerminalBackgroundColor = function(color) {
         $scope.terminalBackgroundColor = color;
     }
 
-    $scope.text_color_for_color = function(color) {
+    $scope.text_color_for_color = text_color_for_color;
-        return text_color_for_color(color);
+    
-    }
+    $scope.border_color_for_color = border_color_for_color;
 
     $scope.getColorArraysArray = function() {
         var result = null;
@@ -39,12 +43,35 @@ controllers.controller("colorsController", function($scope, $http) {
         return result;
     }
     
+    $scope.beginCustomizationWithSetting = function(setting) {
+    	if (! $scope.customizationActive) {
+	    	$scope.customizationActive = true;
+			$scope.selectedColorSetting = setting;
+			$scope.csEditingType = setting;
+			$scope.csUserVisibleTitle = user_visible_title_for_setting_name(setting);
+    	}
+    }
+        
     $scope.selectColorSetting = function(name) {
     	$scope.selectedColorSetting = name;
+    	$scope.csEditingType = $scope.customizationActive ? name : '';
+    	$scope.csUserVisibleTitle = user_visible_title_for_setting_name(name);
+    	$scope.beginCustomizationWithSetting(name);
+    }
+    
+    $scope.toggleCustomizationActive = function() {
+	    if (! $scope.customizationActive) {
+	    	$scope.beginCustomizationWithSetting($scope.selectedColorSetting || 'command');
+	    } else {
+	    	$scope.customizationActive = false;
+			$scope.selectedColorSetting = '';
+			$scope.csEditingType = '';
+	    }
     }
 
     $scope.changeSelectedTextColor = function(color) {
         $scope.selectedColorScheme[$scope.selectedColorSetting] = color;
+        $scope.noteThemeChanged();
     }
 
     $scope.sampleTerminalBackgroundColors = ['white', '#' + solarized.base3, '#300', '#003', '#' + solarized.base03, '#232323', 'black'];
@@ -65,11 +92,25 @@ controllers.controller("colorsController", function($scope, $http) {
             $scope.changeSelectedColorScheme(currentScheme);
      })};
 
+	$scope.saveThemeButtonTitle = "Set Theme";
+	
+	$scope.noteThemeChanged = function() {
+		$scope.saveThemeButtonTitle = "Set Theme";
+	}
+
     $scope.setTheme = function() {
         var settingNames = ["autosuggestion", "command", "param", "redirection", "comment", "error", "quote", "end"];
+        var remaining = settingNames.length;
         for (name in settingNames) {
             var postData = "what=" + settingNames[name] + "&color=" + $scope.selectedColorScheme[settingNames[name]] + "&background_color=&bold=&underline=";
             $http.post("/set_color/", postData, { headers: {'Content-Type': 'application/x-www-form-urlencoded'} }).success(function(data, status, headers, config) {
+            	if (status == 200) {
+            		remaining -= 1;
+            		if (remaining == 0) {
+            			/* All styles set! */
+            			$scope.saveThemeButtonTitle = "Theme Set!";
+            		}
+            	}
             })
         }
     };

share/tools/web_config/partials/colors.html

@@ -1,31 +1,110 @@
 <div>
     <!-- ko with: color_picker -->
-    <span style="padding-left: 25px">Click to customize each color: </span><br>
     <div class="colorpicker_text_sample" ng-style="{'background-color': terminalBackgroundColor}">
-        <span style="position: absolute; left: 10px; top: -6px;" data-ng-style="{'color': text_color_for_color(selectedColorScheme.preferred_background || 'white')}">{{ selectedColorScheme.name }}</span><br>
+        <span style="position: absolute; left: 10px; top: 3px;" data-ng-style="{'color': text_color_for_color(terminalBackgroundColor || 'white')}">{{ selectedColorScheme.name }}</span><br>
         <div class="color_picker_background_cells">
-            <div ng-style="{'background-color': color}" ng-repeat="color in sampleTerminalBackgroundColors" ng-click="changeTerminalBackgroundColor(color)"></div>
+	        <span data-ng-style="{'color': text_color_for_color(terminalBackgroundColor || 'white')}">Background: </span>
+            <div ng-style="{'background-color': color}" ng-repeat="color in sampleTerminalBackgroundColors" ng-click="changeTerminalBackgroundColor(color)"  title="Preview with this background color.
+            
+fish cannot change the background color of your terminal. Refer to your terminal documentation to set its background color."></div>
         </div>
         <!-- This is the sample text -->
-        <span data-ng-style="{ 'color': selectedColorScheme.command}" ng-click="selectColorSetting('command')">/bright/vixens</span>
+        <span ng-class="{cs_clickable: customizationActive, cs_editing: csEditingType == 'command'}"
-        <span data-ng-style="{ 'color': selectedColorScheme.param}" ng-click="selectColorSetting('param')">jump</span>
+        	  ng-mouseenter="csHoveredType = 'command'"
-        <span data-ng-style="{ 'color': selectedColorScheme.end}" ng-click="selectColorSetting('end')">|</span>
+        	  ng-mouseleave="csHoveredType = false"
-        <span data-ng-style="{ 'color': selectedColorScheme.command}" ng-click="selectColorSetting('command')">dozy</span>
+              data-ng-style="{ 'color': selectedColorScheme.command}"
-        <span data-ng-style="{ 'color': selectedColorScheme.quote}" ng-click="selectColorSetting('quote')"> "fowl" </span>
+              ng-click="selectColorSetting('command')">/bright/vixens</span>
-        <span data-ng-style="{ 'color': selectedColorScheme.redirection}" ng-click="selectColorSetting('redirection')">&gt; quack</span>
-        <span data-ng-style="{ 'color': selectedColorScheme.end}" ng-click="selectColorSetting('end')">&</span>
-        <br>
-        <span data-ng-style="{ 'color': selectedColorScheme.command}" ng-click="selectColorSetting('command')">echo</span>
-        <span data-ng-style="{ 'color': selectedColorScheme.error}" ng-click="selectColorSetting('error')">'Errors are the portals to discovery</span>
-        <br>
-        <span data-ng-style="{ 'color': selectedColorScheme.comment}" ng-click="selectColorSetting('comment')"># This is a comment</span>
-        <br>
-        <span data-ng-style="{ 'color': selectedColorScheme.command}" ng-click="selectColorSetting('command')">Th</span><span data-ng-style="{ 'color': selectedColorScheme.autosuggestion }" ng-click="selectColorSetting('autosuggestion')"><span class="fake_cursor"><span style="visibility: hidden">i</span></span>s is an autosuggestion</span>
         
-        <span class="save_button" style="position: absolute; right: 5px; bottom: 5px;" title="Terminal background color is not set automatically on Apply. See your terminal documentation to set its background color." data-ng-style="{'color': text_color_for_color(selectedColorScheme.preferred_background || 'white')}" ng-show="showSaveButton" ng-click="setTheme()">Apply</span>
+        <span ng-class="{cs_clickable: customizationActive, cs_editing: csEditingType == 'param'}"
+        	  ng-mouseenter="csHoveredType = 'param'"
+        	  ng-mouseleave="csHoveredType = false"
+              data-ng-style="{ 'color': selectedColorScheme.param}"
+              ng-click="selectColorSetting('param')">jump</span>
+
+        <span ng-class="{cs_clickable: customizationActive, cs_editing: csEditingType == 'end'}"
+        	  ng-mouseenter="csHoveredType = 'end'"
+        	  ng-mouseleave="csHoveredType = false"
+              data-ng-style="{ 'color': selectedColorScheme.end}"
+              ng-click="selectColorSetting('end')">|</span>
+              
+        <span ng-class="{cs_clickable: customizationActive, cs_editing: csEditingType == 'command'}"
+        	  ng-mouseenter="csHoveredType = 'command'"
+        	  ng-mouseleave="csHoveredType = false"
+              data-ng-style="{ 'color': selectedColorScheme.command}"
+              ng-click="selectColorSetting('command')">dozy</span>
+    	
+        <span ng-class="{cs_clickable: customizationActive, cs_editing: csEditingType == 'quote'}"
+        	  ng-mouseenter="csHoveredType = 'quote'"
+        	  ng-mouseleave="csHoveredType = false"
+              data-ng-style="{ 'color': selectedColorScheme.quote}"
+              ng-click="selectColorSetting('quote')"> "fowl"</span>
+        
+        <span ng-class="{cs_clickable: customizationActive, cs_editing: csEditingType == 'redirection'}"
+        	  ng-mouseenter="csHoveredType = 'redirection'"
+        	  ng-mouseleave="csHoveredType = false"
+              data-ng-style="{ 'color': selectedColorScheme.redirection}"
+              ng-click="selectColorSetting('redirection')">&gt; quack</span>
+        
+        <span ng-class="{cs_clickable: customizationActive, cs_editing: csEditingType == 'end'}"
+        	  ng-mouseenter="csHoveredType = 'end'"
+        	  ng-mouseleave="csHoveredType = false"
+              data-ng-style="{ 'color': selectedColorScheme.end}"
+              ng-click="selectColorSetting('end')">&</span>
+              
+        <br>
+        
+        <span ng-class="{cs_clickable: customizationActive, cs_editing: csEditingType == 'command'}"
+        	  ng-mouseenter="csHoveredType = 'command'"
+        	  ng-mouseleave="csHoveredType = false"
+              data-ng-style="{ 'color': selectedColorScheme.command}"
+              ng-click="selectColorSetting('command')">echo</span>
+        
+        <span ng-class="{cs_clickable: customizationActive, cs_editing: csEditingType == 'error'}"
+        	  ng-mouseenter="csHoveredType = 'error'"
+        	  ng-mouseleave="csHoveredType = false"
+              data-ng-style="{ 'color': selectedColorScheme.error}"
+              ng-click="selectColorSetting('error')">'Errors are the portals to discovery</span>
+    	
+        <br>
+        
+        <span ng-class="{cs_clickable: customizationActive, cs_editing: csEditingType == 'comment'}"
+        	  ng-mouseenter="csHoveredType = 'comment'"
+        	  ng-mouseleave="csHoveredType = false"
+        	  data-ng-style="{ 'color': selectedColorScheme.comment}"
+        	  ng-click="selectColorSetting('comment')"># This is a comment</span>
+        
+        <br>
+        
+        <span ng-class="{cs_clickable: customizationActive, cs_editing: csEditingType == 'command'}"
+        	  ng-mouseenter="csHoveredType = 'command'"
+        	  ng-mouseleave="csHoveredType = false"
+              data-ng-style="{ 'color': selectedColorScheme.command}"
+              ng-click="selectColorSetting('command')">Th</span><span 
+              class="fake_cursor"><span style="visibility: hidden">i</span></span><span
+              ng-class="{cs_clickable: customizationActive, cs_editing: csEditingType == 'autosuggestion'}"
+        	  ng-mouseenter="csHoveredType = 'autosuggestion'"
+        	  ng-mouseleave="csHoveredType = false"
+              data-ng-style="{ 'color': selectedColorScheme.autosuggestion }"
+              ng-click="selectColorSetting('autosuggestion')">s is an autosuggestion</span>
+
+
+		<div style="position: absolute; right: 5px; bottom: 5px;">
+		
+			<span class="customize_theme_button"
+				  ng-class="{button_highlight: customizationActive}"
+				  data-ng-style="{'color': text_color_for_color(terminalBackgroundColor || 'white')}"
+				  ng-click="toggleCustomizationActive();">Customize</span>
+			  
+			<span class="save_button"
+				  data-ng-style="{'color': text_color_for_color(terminalBackgroundColor || 'white')}"
+				  ng-show="showSaveButton"
+				  ng-click="setTheme()">{{saveThemeButtonTitle}}</span>
+		</div>
 
     </div>
             
+    <div ng-show="customizationActive">
+		<div style="margin: 10px 0 7px 35px;">Choose a color for {{csUserVisibleTitle}}:</div>
 		<table class="colorpicker_term256" style="margin: 0px 20px;">
 			<tbody>
 				<tr class="colorpicker_term256_row" data-ng-repeat="color_array in colorArraysArray"> 
@@ -36,12 +115,16 @@
 			</tbody>
 			<!-- /ko -->
 		</table>
+	</div>
     
+    <div style="margin: 10px 0 7px 35px;">Preview a theme below:</div>
+    
+    <div class="color_scheme_choices_scrollview">
 		<div class="color_scheme_choices_list">
 			<div class="color_scheme_choice_container" data-ng-repeat="colorScheme in colorSchemes" ng-click="changeSelectedColorScheme(colorScheme)">
 				<div class="color_scheme_choice_label">
 					<!-- This click/clickBubble nonsense is so that we can have a separate URL inside a parent with an onClick handler -->
-                <span>{{colorScheme.name }}</span><!--a data-bind="if: $data.url, click: function(){return true;}, clickBubble: false, attr: { href: $data.url}"><img class="external_link_img" src="external_link.png"></a-->
+					<span style="color: #AAA">{{colorScheme.name}}</span><!--a data-bind="if: $data.url, click: function(){return true;}, clickBubble: false, attr: { href: $data.url}"><img class="external_link_img" src="external_link.png"></a-->
 				</div>
 				<div class="colorpicker_text_sample_tight" data-ng-style="{'background-color': colorScheme.preferred_background}">
 					<span data-ng-style="{'color': colorScheme.command}">/bright/vixens</span>
@@ -57,7 +140,8 @@
 					<br>
 					<span data-ng-style="{ 'color': colorScheme.comment}"># This is a comment</span>
 					<br>
-                <span data-ng-style="{ 'color': colorScheme.command}">Th</span><span data-ng-style="{ 'color': colorScheme.autosuggestion}"><span class="fake_cursor"><span style="visibility: hidden">i</span></span>s is an autosuggestion</span>
+					<span data-ng-style="{ 'color': colorScheme.command}">Th</span><span class="fake_cursor"><span style="visibility: hidden">i</span></span><span data-ng-style="{ 'color': colorScheme.autosuggestion}">s is an autosuggestion</span>
+				</div>
 			</div>
 		</div>
     </div>

share/tools/web_config/webconfig.py

@@ -291,7 +291,8 @@ class BindingParser:
 
     def get_char(self):
         """ Gets next character from buffer """
-
+        if self.index >= len(self.buffer):
+            return '\0'
         c = self.buffer[self.index]
         self.index += 1
         return c
@@ -811,7 +812,7 @@ PORT = 8000
 while PORT <= 9000:
     try:
         Handler = FishConfigHTTPRequestHandler
-        httpd = SocketServer.TCPServer(("", PORT), Handler)
+        httpd = SocketServer.TCPServer(("127.0.0.1", PORT), Handler)
         # Success
         break
     except socket.error:
@@ -830,7 +831,7 @@ if PORT > 9000:
 # Just look at the first letter
 initial_tab = ''
 if len(sys.argv) > 1:
-    for tab in ['functions', 'prompt', 'colors', 'variables', 'history']:
+    for tab in ['functions', 'prompt', 'colors', 'variables', 'history', 'bindings']:
         if tab.startswith(sys.argv[1]):
             initial_tab = '#' + tab
             break

tests/test.fish

@@ -60,13 +60,15 @@ for i in *.in
   if diff tmp.out $template_out >/dev/null
   else
 	set res fail
-	echo Output differs for file $i
+	echo Output differs for file $i. Diff follows:
+	diff tmp.out $template_out
   end
 
   if diff tmp.err $template_err >/dev/null
   else
 	set res fail
-	echo Error output differs for file $i
+	echo Error output differs for file $i. Diff follows:
+	diff tmp.err $template_err
   end
 
   if test (cat tmp.status) = (cat $template_status)

tests/test1.in

@@ -121,3 +121,12 @@ echo -e Catch your breath
 
 echo -e 'abc\x21def'
 echo -e 'abc\x211def'
+
+function always_fails
+    if true
+        return 1
+    end
+end
+
+always_fails ; echo $status
+

tests/test1.out

@@ -36,3 +36,4 @@ abc
 Catch your breath
 abc!def
 abc!1def
+1

tests/test8.in

@@ -23,3 +23,7 @@ echo (seq $n)[3..5 -2..2]
 echo Test more
 echo $test[(count $test)..1]
 echo $test[1..(count $test)]
+
+# See issue 1061
+echo "Verify that if statements swallow failure"
+if false ; end ; echo $status

tests/test8.out

@@ -15,3 +15,5 @@ Test command substitution
 Test more
 10 9 8 7 6 5 4 3 2 1
 1 2 3 4 5 6 7 8 9 10
+Verify that if statements swallow failure
+0

tests/test9.in

@@ -70,4 +70,18 @@ end
 # Test implicit cd. This should do nothing.
 ./
 
+# Test special for loop expansion
+# Here we the name of the variable is derived from another variable
+echo "Testing for loop"
+set var1 var2
+for $var1 in 1 2 3
+    echo -n $var2
+end
+echo
+
+# Test status -n
+eval 'status -n
+status -n
+status -n'
+
 false