mirror of
https://github.com/fish-shell/fish-shell
synced 2024-12-25 20:33:08 +00:00
Rejigger test suite
Split test_interactive off from test_fishscript and add a new target test_high_level that tests both. Add some Makefile magic so the tests can be run serially without using sub-make, which gets rid of a little noise from the make output. Rewrite interactive tests to look better.
This commit is contained in:
Kevin Ballard
parent
8d03baa4e0
commit
5f82f721d2
7 changed files with 141 additions and 75 deletions
40
Makefile.in
40
Makefile.in
|
|
@ -288,26 +288,42 @@ doc/refman.pdf: doc
|
|||
# This target runs both the low level code tests and the high level script tests.
|
||||
#
|
||||
|
||||
# normally we'd just depend on test_low_level and test_fishscript
|
||||
# But I'd rather not mix test output when doing parallel building with the -j flag
|
||||
# Instead we'll depend on the dependencies of those targets, for parallel building,
|
||||
# and use recursive make to actually invoke the tests themselves
|
||||
test: $(PROGRAMS) fish_tests
|
||||
@$(MAKE) test_low_level
|
||||
@$(MAKE) test_fishscript
|
||||
test: test_low_level test_high_level
|
||||
.PHONY: test
|
||||
|
||||
test_low_level: fish_tests
|
||||
# We want the various tests to run serially so their output doesn't mix
|
||||
# We can do that by adding ordering dependencies based on what goals are being used.
|
||||
|
||||
test_goals := test_low_level test_fishscript test_interactive
|
||||
|
||||
# The following variables define targets that depend on the tests. If any more targets
|
||||
# are added that depend, directly or indirectly, on tests, they need to be recorded here.
|
||||
test_test_deps = test_low_level $(test_high_level_test_deps)
|
||||
test_high_level_test_deps = test_fishscript test_interactive
|
||||
|
||||
active_test_goals = $(filter $(test_goals),$(foreach a,$(or $(MAKECMDGOALS),$(.DEFAULT_GOAL)),$(a) $($(a)_test_deps)))
|
||||
filter_up_to = $(eval b:=1)$(foreach a,$(2),$(and $(b),$(if $(subst $(1),,$(a)),$(a),$(eval b:=))))
|
||||
|
||||
test_low_level: fish_tests $(call filter_up_to,test_low_level,$(active_test_goals))
|
||||
./fish_tests
|
||||
.PHONY: test_low_level
|
||||
|
||||
test_fishscript: $(PROGRAMS)
|
||||
@rm -rf tests/tmp.config
|
||||
@mkdir -p tests/tmp.config/fish
|
||||
@echo 'set fish_function_path "$$PWD/../share/functions"' > tests/tmp.config/fish/config.fish
|
||||
test_high_level: test_fishscript test_interactive
|
||||
.PHONY: test_high_level
|
||||
|
||||
test_fishscript: $(PROGRAMS) test_fishscript_config $(call filter_up_to,test_fishscript,$(active_test_goals))
|
||||
cd tests; XDG_CONFIG_HOME="$$PWD"/tmp.config ../fish test.fish
|
||||
.PHONY: test_fishscript
|
||||
|
||||
test_interactive: $(PROGRAMS) test_fishscript_config $(call filter_up_to,test_interactive,$(active_test_goals))
|
||||
cd tests; XDG_CONFIG_HOME="$$PWD"/tmp.config ../fish interactive.fish
|
||||
.PHONY: test_interactive
|
||||
|
||||
test_fishscript_config:
|
||||
@rm -rf tests/tmp.config
|
||||
@mkdir -p tests/tmp.config/fish
|
||||
@echo 'set fish_function_path "$$PWD/../share/functions"' > tests/tmp.config/fish/config.fish
|
||||
.PHONY: test_fishscript_config
|
||||
|
||||
#
|
||||
# commands.hdr collects documentation on all commands, functions and
|
||||
|
|
|
|||
|
|
@ -2,54 +2,78 @@
|
|||
#
|
||||
# Interactive tests using `expect`
|
||||
|
||||
function die
|
||||
echo $argv[1] >&2
|
||||
exit 1
|
||||
source test_util.fish
|
||||
|
||||
say -o cyan "Testing interactive functionality"
|
||||
if not type -q expect
|
||||
say red "Tests disabled: `expect` not found"
|
||||
exit 0
|
||||
end
|
||||
|
||||
for i in *.expect
|
||||
function test_file
|
||||
rm -Rf tmp.interactive.config; or die "Couldn't remove tmp.interactive.config"
|
||||
mkdir -p tmp.interactive.config/fish; or die "Couldn't create tmp.interactive.config/fish"
|
||||
cp interactive.config tmp.interactive.config/fish/config.fish; or die "Couldn't create tmp.interactive.config/fish/config.fish"
|
||||
|
||||
set -l file $argv[1]
|
||||
|
||||
echo -n "Testing file $file ... "
|
||||
|
||||
begin
|
||||
set -lx XDG_CONFIG_HOME $PWD/tmp.interactive.config
|
||||
set -lx TERM dumb
|
||||
expect -n -c 'source interactive.expect.rc' -f $i >$i.tmp.out ^$i.tmp.err
|
||||
expect -n -c 'source interactive.expect.rc' -f $file >$file.tmp.out ^$file.tmp.err
|
||||
end
|
||||
set -l tmp_status $status
|
||||
set res ok
|
||||
mv -f interactive.tmp.log $i.tmp.log
|
||||
if not diff $i.tmp.out $i.out >/dev/null
|
||||
set res fail
|
||||
echo "Output differs for file $i. Diff follows:"
|
||||
diff -u $i.tmp.out $i.out
|
||||
end
|
||||
set -l res ok
|
||||
mv -f interactive.tmp.log $file.tmp.log
|
||||
|
||||
if not diff $i.tmp.err $i.err >/dev/null
|
||||
set res fail
|
||||
echo "Error output differs for file $i. Diff follows:"
|
||||
diff -u $i.tmp.err $i.err
|
||||
end
|
||||
diff $file.tmp.out $file.out >/dev/null
|
||||
set -l out_status $status
|
||||
diff $file.tmp.err $file.err >/dev/null
|
||||
set -l err_status $status
|
||||
set -l exp_status (cat $file.status)[1]
|
||||
|
||||
set -l exp_status (cat $i.status)[1]
|
||||
if test $tmp_status != $exp_status
|
||||
set res fail
|
||||
echo "Exit status differs for file $i."
|
||||
echo "Expected $exp_status, got $tmp_status."
|
||||
end
|
||||
|
||||
if test $res = ok
|
||||
echo "File $i tested ok"
|
||||
if test $out_status -eq 0 -a $err_status -eq 0 -a $exp_status -eq $tmp_status
|
||||
say green "ok"
|
||||
# clean up tmp files
|
||||
rm -f $i.tmp.{err,out,log}
|
||||
rm -f $file.tmp.{err,out,log}
|
||||
return 0
|
||||
else
|
||||
if set -qgx SHOW_INTERACTIVE_LOG
|
||||
say red "fail"
|
||||
if test $out_status -ne 0
|
||||
say yellow "Output differs for file $file. Diff follows:"
|
||||
colordiff -u $file.tmp.out $file.out
|
||||
end
|
||||
if test $err_status -ne 0
|
||||
say yellow "Error output differs for file $file. Diff follows:"
|
||||
colordiff -u $file.tmp.err $file.err
|
||||
end
|
||||
if test $exp_status -ne $tmp_status
|
||||
say yellow "Exit status differs for file $file."
|
||||
echo "Expected $exp_status, got $tmp_status."
|
||||
end
|
||||
if set -q SHOW_INTERACTIVE_LOG
|
||||
# dump the interactive log
|
||||
# primarily for use in travis where we can't check it manually
|
||||
echo "Log for file $i:"
|
||||
cat $i.tmp.log
|
||||
say yellow "Log for file $file:"
|
||||
cat $file.tmp.log
|
||||
end
|
||||
echo "File $i failed tests"
|
||||
return 1
|
||||
end
|
||||
end
|
||||
|
||||
set -l failed 0
|
||||
for i in *.expect
|
||||
if not test_file $i
|
||||
set failed (expr $failed + 1)
|
||||
end
|
||||
end
|
||||
|
||||
if test $failed -eq 0
|
||||
say green "All tests completed successfully"
|
||||
exit 0
|
||||
else
|
||||
say red "$failed test"(test $failed -eq 1; or echo s)" failed"
|
||||
exit 1
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
File bind.expect tested ok
|
||||
File generic.expect tested ok
|
||||
File read.expect tested ok
|
||||
|
|
@ -1 +0,0 @@
|
|||
0
|
||||
|
|
@ -37,34 +37,6 @@ if [ "$argv" != '-n' ]
|
|||
echo "Profiling failed"
|
||||
end
|
||||
|
||||
echo "Testing interactive functionality"
|
||||
# bug: `fish -n` throws errors on fishscript functions that don't shadow real commands,
|
||||
# so we can't use `type -q expect` here.
|
||||
if command -s expect >/dev/null
|
||||
# we have expect, we can run the interactive tests
|
||||
begin
|
||||
../fish -n ./interactive.fish ^interactive.tmp.err
|
||||
../fish ./interactive.fish ^^interactive.tmp.err
|
||||
end | tee interactive.tmp.out
|
||||
set -l tmp_status $status
|
||||
if not diff interactive.tmp.out interactive.out >/dev/null
|
||||
set res fail
|
||||
echo "Output differs for file interactive.fish"
|
||||
end
|
||||
|
||||
if not diff interactive.tmp.err interactive.err >/dev/null
|
||||
set res fail
|
||||
echo "Error output differs for file interactive.fish"
|
||||
end
|
||||
|
||||
if test $tmp_status -ne (cat interactive.status)
|
||||
set res fail
|
||||
echo "Exit status differs for file interactive.fish"
|
||||
end
|
||||
else
|
||||
echo "Tests disabled: `expect` not found"
|
||||
end
|
||||
|
||||
if test $res = ok
|
||||
echo "File test.fish tested ok"
|
||||
exit 0
|
||||
|
|
|
|||
58
tests/test_util.fish
Normal file
58
tests/test_util.fish
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
# Utilities for the test runners
|
||||
|
||||
function die
|
||||
echo $argv[1] >&2
|
||||
exit 1
|
||||
end
|
||||
|
||||
if not tty 0>&1 >/dev/null
|
||||
function set_color
|
||||
# do nothing
|
||||
return 0
|
||||
end
|
||||
end
|
||||
|
||||
function say
|
||||
set -l color_flags
|
||||
while set -q argv[1]
|
||||
switch $argv[1]
|
||||
case -b -o -u
|
||||
set color_flags $color_flags $argv[1]
|
||||
case --
|
||||
set -e argv[1]
|
||||
break
|
||||
case -\*
|
||||
case \*
|
||||
break
|
||||
end
|
||||
set -e argv[1]
|
||||
end
|
||||
|
||||
if not set -q argv[1]
|
||||
echo 'usage: say [flags] color [string...]' >&2
|
||||
return 1
|
||||
end
|
||||
|
||||
if set_color $color_flags $argv[1]
|
||||
set -e argv[1]
|
||||
echo $argv
|
||||
set -l stat $status
|
||||
set_color reset
|
||||
or return $stat
|
||||
end
|
||||
end
|
||||
|
||||
function colordiff -d 'Colored diff output for unified diffs'
|
||||
diff $argv | while read -l line
|
||||
switch $line
|
||||
case '+*'
|
||||
say green $line
|
||||
case '-*'
|
||||
say red $line
|
||||
case '@*'
|
||||
say cyan $line
|
||||
case '*'
|
||||
echo $line
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Reference in a new issue