Rejigger test suite

Split test_interactive off from test_fishscript and add a new target
test_high_level that tests both.

Add some Makefile magic so the tests can be run serially without using
sub-make, which gets rid of a little noise from the make output.

Rewrite interactive tests to look better.

Makefile.in

@@ -288,26 +288,42 @@ doc/refman.pdf: doc
 # This target runs both the low level code tests and the high level script tests.
 #
 
-# normally we'd just depend on test_low_level and test_fishscript
+test: test_low_level test_high_level
-# But I'd rather not mix test output when doing parallel building with the -j flag
-# Instead we'll depend on the dependencies of those targets, for parallel building,
-# and use recursive make to actually invoke the tests themselves
-test: $(PROGRAMS) fish_tests
-	@$(MAKE) test_low_level
-	@$(MAKE) test_fishscript
 .PHONY: test
 
-test_low_level: fish_tests
+# We want the various tests to run serially so their output doesn't mix
+# We can do that by adding ordering dependencies based on what goals are being used.
+
+test_goals := test_low_level test_fishscript test_interactive
+
+# The following variables define targets that depend on the tests. If any more targets
+# are added that depend, directly or indirectly, on tests, they need to be recorded here.
+test_test_deps = test_low_level $(test_high_level_test_deps)
+test_high_level_test_deps = test_fishscript test_interactive
+
+active_test_goals = $(filter $(test_goals),$(foreach a,$(or $(MAKECMDGOALS),$(.DEFAULT_GOAL)),$(a) $($(a)_test_deps)))
+filter_up_to = $(eval b:=1)$(foreach a,$(2),$(and $(b),$(if $(subst $(1),,$(a)),$(a),$(eval b:=))))
+
+test_low_level: fish_tests $(call filter_up_to,test_low_level,$(active_test_goals))
 	./fish_tests
 .PHONY: test_low_level
 
-test_fishscript: $(PROGRAMS)
+test_high_level: test_fishscript test_interactive
-	@rm -rf tests/tmp.config
+.PHONY: test_high_level
-	@mkdir -p tests/tmp.config/fish
+
-	@echo 'set fish_function_path "$$PWD/../share/functions"' > tests/tmp.config/fish/config.fish
+test_fishscript: $(PROGRAMS) test_fishscript_config $(call filter_up_to,test_fishscript,$(active_test_goals))
 	cd tests; XDG_CONFIG_HOME="$$PWD"/tmp.config ../fish test.fish
 .PHONY: test_fishscript
 
+test_interactive: $(PROGRAMS) test_fishscript_config $(call filter_up_to,test_interactive,$(active_test_goals))
+	cd tests; XDG_CONFIG_HOME="$$PWD"/tmp.config ../fish interactive.fish
+.PHONY: test_interactive
+
+test_fishscript_config:
+	@rm -rf tests/tmp.config
+	@mkdir -p tests/tmp.config/fish
+	@echo 'set fish_function_path "$$PWD/../share/functions"' > tests/tmp.config/fish/config.fish
+.PHONY: test_fishscript_config
 
 #
 # commands.hdr collects documentation on all commands, functions and

tests/interactive.fish

@@ -2,54 +2,78 @@
 #
 # Interactive tests using `expect`
 
-function die
+source test_util.fish
-    echo $argv[1] >&2
+
-    exit 1
+say -o cyan "Testing interactive functionality"
+if not type -q expect
+    say red "Tests disabled: `expect` not found"
+    exit 0
 end
 
-for i in *.expect
+function test_file
     rm -Rf tmp.interactive.config; or die "Couldn't remove tmp.interactive.config"
     mkdir -p tmp.interactive.config/fish; or die "Couldn't create tmp.interactive.config/fish"
     cp interactive.config tmp.interactive.config/fish/config.fish; or die "Couldn't create tmp.interactive.config/fish/config.fish"
 
+    set -l file $argv[1]
+
+    echo -n "Testing file $file ... "
+
     begin
         set -lx XDG_CONFIG_HOME $PWD/tmp.interactive.config
         set -lx TERM dumb
-        expect -n -c 'source interactive.expect.rc' -f $i >$i.tmp.out ^$i.tmp.err
+        expect -n -c 'source interactive.expect.rc' -f $file >$file.tmp.out ^$file.tmp.err
     end
     set -l tmp_status $status
-    set res ok
+    set -l res ok
-    mv -f interactive.tmp.log $i.tmp.log
+    mv -f interactive.tmp.log $file.tmp.log
-    if not diff $i.tmp.out $i.out >/dev/null
-        set res fail
-        echo "Output differs for file $i. Diff follows:"
-        diff -u $i.tmp.out $i.out
-    end
 
-    if not diff $i.tmp.err $i.err >/dev/null
+    diff $file.tmp.out $file.out >/dev/null
-        set res fail
+    set -l out_status $status
-        echo "Error output differs for file $i. Diff follows:"
+    diff $file.tmp.err $file.err >/dev/null
-        diff -u $i.tmp.err $i.err
+    set -l err_status $status
-    end
+    set -l exp_status (cat $file.status)[1]
 
-    set -l exp_status (cat $i.status)[1]
+    if test $out_status -eq 0 -a $err_status -eq 0 -a $exp_status -eq $tmp_status
-    if test $tmp_status != $exp_status
+        say green "ok"
-        set res fail
-        echo "Exit status differs for file $i."
-        echo "Expected $exp_status, got $tmp_status."
-    end
-
-    if test $res = ok
-        echo "File $i tested ok"
         # clean up tmp files
-        rm -f $i.tmp.{err,out,log}
+        rm -f $file.tmp.{err,out,log}
+        return 0
     else
-        if set -qgx SHOW_INTERACTIVE_LOG
+        say red "fail"
+        if test $out_status -ne 0
+            say yellow "Output differs for file $file. Diff follows:"
+            colordiff -u $file.tmp.out $file.out
+        end
+        if test $err_status -ne 0
+            say yellow "Error output differs for file $file. Diff follows:"
+            colordiff -u $file.tmp.err $file.err
+        end
+        if test $exp_status -ne $tmp_status
+            say yellow "Exit status differs for file $file."
+            echo "Expected $exp_status, got $tmp_status."
+        end
+        if set -q SHOW_INTERACTIVE_LOG
             # dump the interactive log
             # primarily for use in travis where we can't check it manually
-            echo "Log for file $i:"
+            say yellow "Log for file $file:"
-            cat $i.tmp.log
+            cat $file.tmp.log
         end
-        echo "File $i failed tests"
+        return 1
     end
 end
+
+set -l failed 0
+for i in *.expect
+    if not test_file $i
+        set failed (expr $failed + 1)
+    end
+end
+
+if test $failed -eq 0
+    say green "All tests completed successfully"
+    exit 0
+else
+    say red "$failed test"(test $failed -eq 1; or echo s)" failed"
+    exit 1
+end

tests/interactive.out

@@ -1,3 +0,0 @@
-File bind.expect tested ok
-File generic.expect tested ok
-File read.expect tested ok

tests/interactive.status

@@ -1 +0,0 @@
-0

tests/test.fish

@@ -37,34 +37,6 @@ if [ "$argv" != '-n' ]
         echo "Profiling failed"
     end
 
-    echo "Testing interactive functionality"
-    # bug: `fish -n` throws errors on fishscript functions that don't shadow real commands,
-    # so we can't use `type -q expect` here.
-    if command -s expect >/dev/null
-        # we have expect, we can run the interactive tests
-        begin
-            ../fish -n ./interactive.fish ^interactive.tmp.err
-            ../fish ./interactive.fish ^^interactive.tmp.err
-        end  | tee interactive.tmp.out
-        set -l tmp_status $status
-        if not diff interactive.tmp.out interactive.out >/dev/null
-            set res fail
-            echo "Output differs for file interactive.fish"
-        end
-
-        if not diff interactive.tmp.err interactive.err >/dev/null
-            set res fail
-            echo "Error output differs for file interactive.fish"
-        end
-
-        if test $tmp_status -ne (cat interactive.status)
-            set res fail
-            echo "Exit status differs for file interactive.fish"
-        end
-    else
-        echo "Tests disabled: `expect` not found"
-    end
-
     if test $res = ok
         echo "File test.fish tested ok"
         exit 0

tests/test_util.fish

@@ -0,0 +1,58 @@
+# Utilities for the test runners
+
+function die
+    echo $argv[1] >&2
+    exit 1
+end
+
+if not tty 0>&1 >/dev/null
+    function set_color
+        # do nothing
+        return 0
+    end
+end
+
+function say
+    set -l color_flags
+    while set -q argv[1]
+        switch $argv[1]
+            case -b -o -u
+                set color_flags $color_flags $argv[1]
+            case --
+                set -e argv[1]
+                break
+            case -\*
+            case \*
+                break
+        end
+        set -e argv[1]
+    end
+
+    if not set -q argv[1]
+        echo 'usage: say [flags] color [string...]' >&2
+        return 1
+    end
+
+    if set_color $color_flags $argv[1]
+        set -e argv[1]
+        echo $argv
+        set -l stat $status
+        set_color reset
+        or return $stat
+    end
+end
+
+function colordiff -d 'Colored diff output for unified diffs'
+    diff $argv | while read -l line
+        switch $line
+            case '+*'
+                say green $line
+            case '-*'
+                say red $line
+            case '@*'
+                say cyan $line
+            case '*'
+                echo $line
+        end
+    end
+end