mirror of
https://github.com/fish-shell/fish-shell
synced 2025-01-25 19:25:06 +00:00
Rewrite test driver in python (#11028)
This replaces the test_driver.sh/test.fish/interactive.fish system with a test driver written in python that calls into littlecheck directly and runs pexpect in a subprocess. This means we reduce the reliance on the fish that we're testing, and we remove a posix sh script that is a weird stumbling block (see my recent quest to make it work on directories with spaces). To run specific tests, e.g. all the tmux tests and bind.py: tests/test_driver.py target/release/ tests/checks/tmux*.fish tests/pexpects/bind.py
This commit is contained in:
Fabian Boehm
GitHub
parent
51adba6ee0
commit
b43b0e0195
18 changed files with 292 additions and 461 deletions
|
|
@ -198,10 +198,11 @@ The tests can be found in three places:
|
|||
When in doubt, the bulk of the tests should be added as a littlecheck test in tests/checks, as they are the easiest to modify and run, and much faster and more dependable than pexpect tests. The syntax is fairly self-explanatory. It's a fish script with the expected output in ``# CHECK:`` or ``# CHECKERR:`` (for stderr) comments.
|
||||
If your littlecheck test has a specific dependency, use ``# REQUIRE: ...`` with a posix sh script.
|
||||
|
||||
Tests are run in a temporary $HOME, but that is shared among the tests by default. If you need a temporary directory for your test, you should create one (e.g. with ``mktemp``).
|
||||
|
||||
The pexpects are written in python and can simulate input and output to/from a terminal, so they are needed for anything that needs actual interactivity. The runner is in tests/pexpect_helper.py, in case you need to modify something there.
|
||||
|
||||
These tests can be run via the tests/test_driver.py python script, which will set up the environment.
|
||||
It sets up a temporary $HOME and also uses it as the current directory, so you do not need to create a temporary directoy in them.
|
||||
|
||||
If you need a command to do something weird to test something, maybe add it to the ``fish_test_helper`` binary (in tests/fish_test_helper.c), or see if it can already do it.
|
||||
|
||||
Local testing
|
||||
|
|
@ -217,11 +218,13 @@ The tests can be run on your local computer on all operating systems.
|
|||
Or you can run them on a fish, without involving cmake::
|
||||
|
||||
cargo build
|
||||
FISHDIR=target/debug tests/test_driver.sh tests/test.fish # script tests, the checks
|
||||
FISHDIR=target/debug tests/test_driver.sh tests/interactive.fish # interactive tests, the pexpects
|
||||
cargo test # for the unit tests
|
||||
tests/test_driver.py --cachedir=/tmp target/debug # for the script and interactive tests
|
||||
|
||||
Here, ``FISHDIR`` refers to a directory with ``fish``, ``fish_indent`` and ``fish_key_reader`` in it.
|
||||
Here, the first argument to test_driver.py refers to a directory with ``fish``, ``fish_indent`` and ``fish_key_reader`` in it.
|
||||
In this example we're in the root of the git repo and have run ``cargo build`` without ``--release``, so it's a debug build.
|
||||
The ``--cachedir /tmp`` argument means it will keep the fish_test_helper binary in /tmp instead of recompiling it for every test.
|
||||
This saves some time, but isn't strictly necessary.
|
||||
|
||||
Git hooks
|
||||
---------
|
||||
|
|
|
|||
|
|
@ -114,8 +114,8 @@ foreach(CHECK ${FISH_CHECKS})
|
|||
get_filename_component(CHECK_NAME ${CHECK} NAME)
|
||||
get_filename_component(CHECK ${CHECK} NAME_WE)
|
||||
add_test(NAME ${CHECK_NAME}
|
||||
COMMAND env FISHDIR=${CMAKE_CURRENT_BINARY_DIR}/ ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.sh
|
||||
${CMAKE_CURRENT_BINARY_DIR}/tests/test.fish ${CHECK}
|
||||
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.py --cachedir ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}
|
||||
checks/${CHECK}.fish
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests
|
||||
)
|
||||
set_tests_properties(${CHECK_NAME} PROPERTIES SKIP_RETURN_CODE ${SKIP_RETURN_CODE})
|
||||
|
|
@ -127,8 +127,8 @@ FILE(GLOB PEXPECTS CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/tests/pexpects/*.py)
|
|||
foreach(PEXPECT ${PEXPECTS})
|
||||
get_filename_component(PEXPECT ${PEXPECT} NAME)
|
||||
add_test(NAME ${PEXPECT}
|
||||
COMMAND env FISHDIR=${CMAKE_CURRENT_BINARY_DIR}/ ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.sh
|
||||
${CMAKE_CURRENT_BINARY_DIR}/tests/interactive.fish ${PEXPECT}
|
||||
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.py --cachedir ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}
|
||||
pexpects/${PEXPECT}
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests
|
||||
)
|
||||
set_tests_properties(${PEXPECT} PROPERTIES SKIP_RETURN_CODE ${SKIP_RETURN_CODE})
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#RUN: fish=%fish %fish %s | %fish %filter-control-sequences
|
||||
#RUN: fish=%fish %fish %s
|
||||
set -g PATH
|
||||
$fish -c "nonexistent-command-1234 banana rama"
|
||||
#CHECKERR: fish: Unknown command: nonexistent-command-1234
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#RUN: %fish --interactive %s | %fish %filter-control-sequences
|
||||
#RUN: %fish --interactive %s
|
||||
# ^ interactive so we can do `complete`
|
||||
mkdir -p __fish_complete_directories/
|
||||
cd __fish_complete_directories
|
||||
|
|
|
|||
|
|
@ -150,7 +150,10 @@ end
|
|||
# CHECK: disown
|
||||
# CHECK: fg
|
||||
# CHECK: fish_command_not_found
|
||||
# CHECK: fish_prompt
|
||||
# CHECK: fish_prompt_event
|
||||
# CHECK: fish_sigtrap_handler
|
||||
# CHECK: fish_title
|
||||
# CHECK: frob
|
||||
# CHECK: kill
|
||||
# CHECK: name1
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#RUN: %fish -i %s | %fish %filter-control-sequences
|
||||
#RUN: %fish -i %s
|
||||
# Note: ^ this is interactive so we test interactive behavior,
|
||||
# e.g. the fish_git_prompt variable handlers test `status is-interactive`.
|
||||
#REQUIRES: command -v git
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#RUN: fish=%fish %fish %s | %fish %filter-control-sequences
|
||||
#RUN: fish=%fish %fish %s
|
||||
|
||||
$fish -c "echo 1.2.3.4."
|
||||
# CHECK: 1.2.3.4.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# RUN: fish=%fish filter_ctrls=%filter-control-sequences %fish %s
|
||||
# RUN: fish=%fish %fish %s
|
||||
# Set term again explicitly to ensure behavior.
|
||||
set -gx TERM xterm
|
||||
# Read with no vars is not an error
|
||||
|
|
@ -248,7 +248,7 @@ if test (string length "$x") -ne $fish_read_limit
|
|||
end
|
||||
|
||||
# Confirm reading non-interactively works -- \#4206 regression
|
||||
echo abc\ndef | $fish -i -c 'read a; read b; set --show a; set --show b' | $fish $filter_ctrls
|
||||
echo abc\ndef | $fish -i -c 'read a; read b; set --show a; set --show b'
|
||||
#CHECK: $a: set in global scope, unexported, with 1 elements
|
||||
#CHECK: $a[1]: |abc|
|
||||
#CHECK: $b: set in global scope, unexported, with 1 elements
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#RUN: fish=%fish filter_ctrls=%filter-control-sequences %fish %s
|
||||
#RUN: fish=%fish %fish %s
|
||||
# Some tests of the "return" builtin.
|
||||
|
||||
$fish -c 'return 5'
|
||||
|
|
@ -21,7 +21,7 @@ begin
|
|||
# but not bar
|
||||
echo $status
|
||||
# CHECK: 69
|
||||
end | $fish $filter_ctrls
|
||||
end
|
||||
|
||||
# Verify negative return values don't cause UB and never map to 0
|
||||
function empty_return
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# RUN: env FISH=%fish filter_ctrls=%filter-control-sequences %fish %s
|
||||
# RUN: env FISH=%fish %fish %s
|
||||
# Environment variable tests
|
||||
|
||||
# Test if variables can be properly set
|
||||
|
|
@ -367,7 +367,7 @@ begin
|
|||
env SHLVL=" 3" $FISH -ic 'echo SHLVL: $SHLVL'
|
||||
# CHECK: SHLVL: 4
|
||||
# CHECK: SHLVL: 4
|
||||
end | $FISH $filter_ctrls
|
||||
end
|
||||
|
||||
# Non-interactive fish doesn't touch $SHLVL
|
||||
env SHLVL=2 $FISH -c 'echo SHLVL: $SHLVL'
|
||||
|
|
|
|||
|
|
@ -1,14 +0,0 @@
|
|||
# Remove the sorts of escape sequences interactive fish prints.
|
||||
|
||||
# First the enable sequences, then a "|" combiner and then the disable ones
|
||||
set -l escapes "\e\[\?2004h"\
|
||||
"\e\[>4;1m"\
|
||||
"\e\[>5u"\
|
||||
"\e="\
|
||||
"|"\
|
||||
"\e\[\?2004l"\
|
||||
"\e\[>4;0m"\
|
||||
"\e\[<1u"\
|
||||
"\e>"
|
||||
|
||||
cat | string replace -ra -- $escapes ''
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
#! /bin/echo "interactive.fish must be run via the test driver!"
|
||||
#
|
||||
# Interactive tests using `pexpect`
|
||||
|
||||
# Set this var to modify behavior of the code being tests. Such as avoiding running
|
||||
# `fish_update_completions` when running tests.
|
||||
set -gx FISH_UNIT_TESTS_RUNNING 1
|
||||
|
||||
# Save the directory containing this script
|
||||
# Do not *cd* here, otherwise you'll ruin our nice tmpdir setup!!!
|
||||
set -l scriptdir (status dirname)
|
||||
|
||||
# Test files specified on commandline, or all pexpect files.
|
||||
if set -q argv[1] && test -n "$argv[1]"
|
||||
set pexpect_files_to_test $scriptdir/pexpects/$argv
|
||||
else if set -q FISH_PEXPECT_FILES
|
||||
set pexpect_files_to_test (string replace -r '^.*/(?=pexpects/)' '' -- $FISH_PEXPECT_FILES)
|
||||
else
|
||||
say -o cyan "Testing interactive functionality"
|
||||
set pexpect_files_to_test $scriptdir/pexpects/*.py
|
||||
end
|
||||
|
||||
source $scriptdir/test_util.fish || exit
|
||||
cat $scriptdir/interactive.config >>$XDG_CONFIG_HOME/fish/config.fish
|
||||
set -lx --prepend PYTHONPATH (realpath $scriptdir)
|
||||
|
||||
function test_pexpect_file
|
||||
set -l file $argv[1]
|
||||
echo -n "Testing file $file:"
|
||||
|
||||
begin
|
||||
set starttime (timestamp)
|
||||
set -lx TERM dumb
|
||||
|
||||
# Help the script find the pexpect_helper module in our parent directory.
|
||||
set -q FISHDIR
|
||||
or set -l FISHDIR ../test/root/bin/
|
||||
set -lx fish $FISHDIR/fish
|
||||
set -lx fish_key_reader $FISHDIR/fish_key_reader
|
||||
path is -fx -- $FISHDIR/fish_test_helper
|
||||
and set -lx fish_test_helper $FISHDIR/fish_test_helper
|
||||
|
||||
# Note we require Python3.
|
||||
python3 $file
|
||||
end
|
||||
|
||||
set -l exit_status $status
|
||||
if test "$exit_status" -eq 0
|
||||
set test_duration (delta $starttime)
|
||||
say green "ok ($test_duration $unit)"
|
||||
else if test "$exit_status" -eq 127
|
||||
say blue "SKIPPED"
|
||||
set exit_status 0
|
||||
end
|
||||
return $exit_status
|
||||
end
|
||||
|
||||
set failed
|
||||
|
||||
# The test here looks wrong, but False sets exit status to 0, which is what we want
|
||||
if python3 -c 'import sys; exit(sys.version_info > (3, 5))'
|
||||
say red "pexpect tests disabled: python3 is too old"
|
||||
set pexpect_files_to_test
|
||||
end
|
||||
if not python3 -c 'import pexpect'
|
||||
say red "pexpect tests disabled: `python3 -c 'import pexpect'` failed"
|
||||
set pexpect_files_to_test
|
||||
end
|
||||
for i in $pexpect_files_to_test
|
||||
if not test_pexpect_file $i
|
||||
# Retry pexpect tests under CI twice, as they are timing-sensitive and CI resource
|
||||
# contention can cause tests to spuriously fail.
|
||||
if set -qx CI
|
||||
say yellow "Trying $i for a second time"
|
||||
if not test_pexpect_file $i
|
||||
set failed $failed $i
|
||||
end
|
||||
else
|
||||
set failed $failed $i
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
set failed (count $failed)
|
||||
if test $failed -eq 0
|
||||
if test (count $pexpect_files_to_test) -gt 1
|
||||
say green "All interactive tests completed successfully"
|
||||
else
|
||||
say green "$pexpect_files_to_test completed successfully"
|
||||
end
|
||||
exit 0
|
||||
else
|
||||
set plural (test $failed -eq 1; or echo s)
|
||||
say red "$failed test$plural failed"
|
||||
exit 1
|
||||
end
|
||||
|
|
@ -11,22 +11,21 @@ send, sendline, sleep, expect_prompt, expect_re, expect_str = (
|
|||
sp.expect_re,
|
||||
sp.expect_str,
|
||||
)
|
||||
expect_prompt()
|
||||
|
||||
# We're going to use three history files, including the default, to verify
|
||||
# that the fish_history variable works as expected.
|
||||
default_histfile = "../test/data/fish/fish_history"
|
||||
my_histfile = "../test/data/fish/my_history"
|
||||
env_histfile = "../test/data/fish/env_history"
|
||||
# (using as a relative path to avoid interpolating $XDG_DATA_HOME)
|
||||
default_histfile = "xdg_data_home/fish/fish_history"
|
||||
my_histfile = "xdg_data_home/fish/my_history"
|
||||
env_histfile = "xdg_data_home/fish/env_history"
|
||||
|
||||
|
||||
def grephistfile(line, file):
|
||||
sendline("grep '^" + line + "' " + file)
|
||||
|
||||
|
||||
# Verify that if we spawn fish with no fish_history env var it uses the
|
||||
# default file.
|
||||
expect_prompt
|
||||
expect_prompt()
|
||||
|
||||
# Verify that a command is recorded in the default history file.
|
||||
cmd1 = "echo $fish_pid default histfile"
|
||||
|
|
|
|||
|
|
@ -1,71 +0,0 @@
|
|||
# Fishscript tests
|
||||
#
|
||||
# There is no shebang line because you shouldn't be running this by hand. You
|
||||
# should be running it via `make test` to ensure the environment is properly
|
||||
# setup.
|
||||
|
||||
# Set this var to modify behavior of the code being tests. Such as avoiding running
|
||||
# `fish_update_completions` when running tests.
|
||||
set -x FISH_UNIT_TESTS_RUNNING 1
|
||||
|
||||
# Save the directory containing this script
|
||||
# Do not *cd* here, otherwise you'll ruin our nice tmpdir setup!!!
|
||||
set -l scriptdir (status dirname)
|
||||
|
||||
# Test files specified on commandline, or all checks.
|
||||
set -l files_to_test
|
||||
if set -q argv[1]
|
||||
set files_to_test $scriptdir/checks/$argv.fish
|
||||
else
|
||||
set files_to_test $scriptdir/checks/*.fish
|
||||
end
|
||||
|
||||
# Be less verbose when running tests one-by-one
|
||||
if test (count $files_to_test) -gt 1
|
||||
say -o cyan "Testing high level script functionality"
|
||||
end
|
||||
|
||||
set -g python (__fish_anypython)
|
||||
or begin
|
||||
say red "Python is not installed. These tests require python."
|
||||
exit 125
|
||||
end
|
||||
|
||||
set -q FISHDIR
|
||||
or set -l FISHDIR ../test/root/bin
|
||||
|
||||
# Test littlecheck files.
|
||||
set -l skipped 0
|
||||
set -l failed 0
|
||||
if set -q files_to_test[1]
|
||||
set -l force_color
|
||||
test "$FISH_FORCE_COLOR" = 1
|
||||
and set force_color --force-color
|
||||
|
||||
$python -S $scriptdir/littlecheck.py \
|
||||
--progress $force_color \
|
||||
-s fish=$FISHDIR/fish \
|
||||
-s fish_test_helper=$fish_test_helper \
|
||||
-s filter-control-sequences="$scriptdir/filter-control-sequences.fish" \
|
||||
$files_to_test
|
||||
|
||||
set -l littlecheck_status $status
|
||||
if test "$littlecheck_status" -eq 125
|
||||
# 125 indicates that all tests executed were skipped.
|
||||
set skipped (count $files_to_test)
|
||||
else
|
||||
# The return code indicates the number of tests that failed
|
||||
set failed $littlecheck_status
|
||||
end
|
||||
end
|
||||
|
||||
if test $failed -eq 0 && test $skipped -gt 0
|
||||
test (count $files_to_test) -gt 1 && say blue (count $files_to_test)" tests skipped"
|
||||
exit 125
|
||||
else if test $failed -eq 0
|
||||
test (count $files_to_test) -gt 1 && say green "All high level script tests completed successfully"
|
||||
exit 0
|
||||
else
|
||||
test (count $files_to_test) -gt 1 && say red "$failed tests failed"
|
||||
exit 1
|
||||
end
|
||||
262
tests/test_driver.py
Executable file
262
tests/test_driver.py
Executable file
|
|
@ -0,0 +1,262 @@
|
|||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import littlecheck
|
||||
|
||||
try:
|
||||
import pexpect
|
||||
|
||||
PEXPECT = True
|
||||
except ImportError:
|
||||
PEXPECT = False
|
||||
|
||||
RESET = "\033[0m"
|
||||
GREEN = "\033[32m"
|
||||
BLUE = "\033[34m"
|
||||
RED = "\033[31m"
|
||||
|
||||
|
||||
def makeenv(script_path, home, test_helper_path):
|
||||
xdg_config = home + "/xdg_config_home"
|
||||
func_dir = xdg_config + "/fish/functions"
|
||||
os.makedirs(func_dir)
|
||||
os.makedirs(xdg_config + "/fish/conf.d/")
|
||||
for func in (script_path / "test_functions").glob("*.fish"):
|
||||
shutil.copy(func, func_dir + "/" + func.parts[-1])
|
||||
shutil.copy(
|
||||
script_path / "interactive.config", xdg_config + "/fish/conf.d/interactive.fish"
|
||||
)
|
||||
|
||||
xdg_data = home + "/xdg_data_home"
|
||||
os.makedirs(xdg_data)
|
||||
xdg_runtime = home + "/xdg_runtime_home"
|
||||
os.makedirs(xdg_runtime)
|
||||
xdg_cache = home + "/xdg_cache_home"
|
||||
os.makedirs(xdg_cache)
|
||||
tmp = home + "/temp"
|
||||
os.makedirs(tmp)
|
||||
|
||||
# Compile fish_test_helper if necessary.
|
||||
# If we're run multiple times, allow keeping this around to save time.
|
||||
if test_helper_path:
|
||||
thp = Path(test_helper_path)
|
||||
if not os.path.exists(thp / "fish_test_helper"):
|
||||
comp = subprocess.run(
|
||||
[
|
||||
"cc",
|
||||
script_path / "fish_test_helper.c",
|
||||
"-o",
|
||||
thp / "fish_test_helper",
|
||||
]
|
||||
)
|
||||
shutil.copy(thp / "fish_test_helper", home + "/fish_test_helper")
|
||||
else:
|
||||
comp = subprocess.run(
|
||||
[
|
||||
"cc",
|
||||
script_path / "fish_test_helper.c",
|
||||
"-o",
|
||||
home + "/fish_test_helper",
|
||||
]
|
||||
)
|
||||
|
||||
# unset LANG, TERM, ...
|
||||
for var in [
|
||||
"XDG_DATA_DIRS",
|
||||
"LANGUAGE",
|
||||
"COLORTERM",
|
||||
"KONSOLE_PROFILE_NAME",
|
||||
"KONSOLE_VERSION",
|
||||
"TERM_PROGRAM",
|
||||
"TERM_PROGRAM_VERSION",
|
||||
"VTE_VERSION",
|
||||
]:
|
||||
if var in os.environ:
|
||||
del os.environ[var]
|
||||
langvars = [key for key in os.environ.keys() if key.startswith("LC_")]
|
||||
for key in langvars:
|
||||
del os.environ[key]
|
||||
|
||||
os.environ.update(
|
||||
{
|
||||
"HOME": home,
|
||||
"TMPDIR": tmp,
|
||||
"FISH_FAST_FAIL": "1",
|
||||
"FISH_UNIT_TESTS_RUNNING": "1",
|
||||
"XDG_CONFIG_HOME": xdg_config,
|
||||
"XDG_DATA_HOME": xdg_data,
|
||||
"XDG_RUNTIME_DIR": xdg_runtime,
|
||||
"XDG_CACHE_HOME": xdg_cache,
|
||||
"fish_test_helper": home + "/fish_test_helper",
|
||||
"TERM": "xterm",
|
||||
"LANG": "C",
|
||||
"LC_CTYPE": "en_US.UTF-8",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: test_driver.py FISH_DIRECTORY TESTS")
|
||||
return 1
|
||||
|
||||
script_path = Path(__file__).parent
|
||||
|
||||
argparser = argparse.ArgumentParser(
|
||||
description="test_driver: Run fish's test suite"
|
||||
)
|
||||
argparser.add_argument(
|
||||
"-f",
|
||||
"--cachedir",
|
||||
type=str,
|
||||
help="Path to keep outputs to speed up the next run",
|
||||
action="store",
|
||||
default=None,
|
||||
)
|
||||
argparser.add_argument("fish", nargs=1, help="Fish to test")
|
||||
argparser.add_argument("file", nargs="*", help="Tests to run")
|
||||
args=argparser.parse_args()
|
||||
|
||||
fishdir = Path(args.fish[0]).absolute()
|
||||
if not fishdir.is_dir():
|
||||
fishdir = fishdir.parent
|
||||
|
||||
failcount = 0
|
||||
failed=[]
|
||||
passcount = 0
|
||||
skipcount = 0
|
||||
def_subs = {"%": "%"}
|
||||
lconfig = littlecheck.Config()
|
||||
lconfig.colorize = sys.stdout.isatty()
|
||||
lconfig.progress = True
|
||||
|
||||
for bin in ["fish", "fish_indent", "fish_key_reader"]:
|
||||
if os.path.exists(fishdir / bin):
|
||||
def_subs[bin] = str(fishdir / bin)
|
||||
else:
|
||||
print(f"Binary does not exist: {fishdir / bin}")
|
||||
return 127
|
||||
|
||||
if args.file:
|
||||
files = [(os.path.abspath(path), path) for path in args.file]
|
||||
else:
|
||||
files = [
|
||||
(os.path.abspath(path), str(path.relative_to(script_path)))
|
||||
for path in sorted(script_path.glob("checks/*.fish"))
|
||||
]
|
||||
files += [
|
||||
(os.path.abspath(path), str(path.relative_to(script_path)))
|
||||
for path in sorted(script_path.glob("pexpects/*.py"))
|
||||
]
|
||||
|
||||
if not PEXPECT and any(x.endswith(".py") for (x, _) in files):
|
||||
print(f"{RED}Skipping pexpect tests because pexpect is not installed{RESET}")
|
||||
|
||||
for f, arg in files:
|
||||
if not f.endswith(".fish") and not f.endswith(".py"):
|
||||
print(f"Not a valid test file: {arg}")
|
||||
failcount += 1
|
||||
continue
|
||||
|
||||
starttime = datetime.now()
|
||||
with tempfile.TemporaryDirectory(prefix="fishtest-") as home:
|
||||
makeenv(script_path, home, args.cachedir)
|
||||
os.chdir(home)
|
||||
if f.endswith(".fish"):
|
||||
subs = def_subs.copy()
|
||||
subs.update({"s": f, "fish_test_helper": home + "/fish_test_helper"})
|
||||
|
||||
# littlecheck
|
||||
print(f"{arg}..", end="", flush=True)
|
||||
ret = littlecheck.check_path(
|
||||
f, subs, lconfig, lambda x: print(x.message())
|
||||
)
|
||||
endtime = datetime.now()
|
||||
duration_ms = round((endtime - starttime).total_seconds() * 1000)
|
||||
if ret is littlecheck.SKIP:
|
||||
print(f"{BLUE}SKIPPED{RESET}")
|
||||
skipcount += 1
|
||||
elif ret:
|
||||
print(f"{GREEN}PASS{RESET} ({duration_ms} ms)")
|
||||
passcount += 1
|
||||
else:
|
||||
print(f"{RED}FAIL{RESET} ({duration_ms} ms)")
|
||||
failcount += 1
|
||||
failed += [arg]
|
||||
print(f"Tmpdir is {home}")
|
||||
elif f.endswith(".py"):
|
||||
# environ for py files has a few changes.
|
||||
pyenviron = os.environ.copy()
|
||||
pyenviron.update(
|
||||
{
|
||||
"PYTHONPATH": str(script_path),
|
||||
"fish": str(fishdir / "fish"),
|
||||
"fish_key_reader": str(fishdir / "fish_key_reader"),
|
||||
"fish_indent": str(fishdir / "fish_indent"),
|
||||
"TERM": "dumb",
|
||||
"FISH_FORCE_COLOR": "1" if sys.stdout.isatty() else "0",
|
||||
}
|
||||
)
|
||||
print(f"{arg}..", end="", flush=True)
|
||||
if not PEXPECT:
|
||||
print(f"{BLUE}SKIPPED{RESET}")
|
||||
skipcount += 1
|
||||
continue
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["python3", f],
|
||||
capture_output=True,
|
||||
env=pyenviron,
|
||||
# Timeout of 120 seconds, about 10 times what any of these takes
|
||||
timeout=120,
|
||||
)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
print(f"{RED}FAILED due to timeout{RESET}")
|
||||
if e.output:
|
||||
print(e.output.decode("utf-8"))
|
||||
if e.stderr:
|
||||
print(e.stderr.decode("utf-8"))
|
||||
failcount += 1
|
||||
failed += [arg]
|
||||
continue
|
||||
|
||||
endtime = datetime.now()
|
||||
duration_ms = round((endtime - starttime).total_seconds() * 1000)
|
||||
if proc.returncode == 0:
|
||||
print(f"{GREEN}PASS{RESET} ({duration_ms} ms)")
|
||||
passcount += 1
|
||||
elif proc.returncode == 127:
|
||||
print(f"{BLUE}SKIPPED{RESET}")
|
||||
skipcount += 1
|
||||
else:
|
||||
print(f"{RED}FAILED{RESET} ({duration_ms} ms)")
|
||||
if proc.stdout:
|
||||
print(proc.stdout.decode("utf-8"))
|
||||
if proc.stderr:
|
||||
print(proc.stderr.decode("utf-8"))
|
||||
failcount += 1
|
||||
failed += [arg]
|
||||
print(f"Tmpdir is {home}")
|
||||
if passcount + failcount + skipcount > 1:
|
||||
print(f"{passcount} / {passcount + failcount} passed ({skipcount} skipped)")
|
||||
if failcount:
|
||||
failstr = '\n '.join(failed)
|
||||
print(f"{RED}Failed tests{RESET}: \n {failstr}")
|
||||
if passcount == 0 and failcount == 0 and skipcount:
|
||||
return 125
|
||||
return 1 if failcount else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
ret = main()
|
||||
sys.exit(ret)
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(130)
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
#!/bin/sh
|
||||
# vim: set ts=4 sw=4 tw=100 et:
|
||||
|
||||
# POSIX sh test driver to reduce dependency on fish in tests.
|
||||
# Executes the specified *fish script* with the provided arguments, after setting up a clean test
|
||||
# environment (see `test_env.sh`) and then importing the fish-related helper functions and
|
||||
# performing some state initialization required by the various fish tests. Each payload script is
|
||||
# executed in its own environment, this script waits for fish to exit then cleans up the target
|
||||
# environment and bubbles up the fish exit code.
|
||||
|
||||
# macOS has really weird default IFS behavior that splits output in random places, and the trailing
|
||||
# backspace is to prevent \n from being gobbled up by the subshell output substitution.
|
||||
# Folks, this is why you should use fish!
|
||||
IFS="$(printf "\n\b")"
|
||||
|
||||
# If CDPATH is set, `cd foo` will print the directory.
|
||||
unset CDPATH
|
||||
|
||||
# The first argument is the path to the script to launch; all remaining arguments are forwarded to
|
||||
# the script.
|
||||
# Resolve the script now because we are going to `cd` later.
|
||||
fish_script="$(realpath $1)"
|
||||
shift 1
|
||||
|
||||
die() {
|
||||
if test "$#" -ge 0; then
|
||||
printf "%s\n" "$@" 1>&2
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
# To keep things sane and to make error messages comprehensible, do not use relative paths anywhere
|
||||
# in this script. Instead, make all paths relative to one of these or the new $HOME ($homedir)."
|
||||
TESTS_ROOT="$(cd $(dirname "$0") && pwd -P)"
|
||||
BUILD_ROOT="$(cd $(dirname "$TESTS_ROOT") && pwd -P)"
|
||||
|
||||
if test -z "$FISHDIR"; then
|
||||
die "Please set \$FISHDIR to a directory that contains fish, fish_indent and fish_key_reader"
|
||||
fi
|
||||
|
||||
FISHDIR=$(realpath -- "$FISHDIR")
|
||||
fish="${FISHDIR}/fish"
|
||||
|
||||
if ! test -x "$fish" || ! test -f "$fish"; then
|
||||
printf '%s\n' "'$fish' is not an executable fish." \
|
||||
"Please set \$FISHDIR to a directory that contains fish, fish_indent and fish_key_reader" >&2
|
||||
exit 7
|
||||
fi
|
||||
|
||||
if ! test -z "$__fish_is_running_tests"; then
|
||||
echo "Recursive test invocation detected!" 1>&2
|
||||
exit 10
|
||||
fi
|
||||
|
||||
# Set up the test environment. Does not change the current working directory.
|
||||
. ${TESTS_ROOT}/test_env.sh
|
||||
|
||||
test -n "$homedir" || die "Failed to set up home"
|
||||
|
||||
# Compile our fish_test_helper program now.
|
||||
# This takes about 50ms.
|
||||
if command -v cc >/dev/null ; then
|
||||
cc "$TESTS_ROOT/fish_test_helper.c" -o "$homedir/fish_test_helper"
|
||||
else
|
||||
echo "Cannot find a C compiler. Skipping tests that require fish_test_helper" >&2
|
||||
fi
|
||||
|
||||
# These are used read-only so it's OK to symlink instead of copy
|
||||
rm -f "$XDG_CONFIG_HOME/fish/functions"
|
||||
ln -s "${TESTS_ROOT}/test_functions" "$XDG_CONFIG_HOME/fish/functions" || die "Failed to symlink"
|
||||
|
||||
# Set the function path at startup, referencing the default fish functions and the test-specific
|
||||
# functions.
|
||||
fish_init_cmd="set fish_function_path '${XDG_CONFIG_HOME}/fish/functions' '${BUILD_ROOT}/share/functions'"
|
||||
|
||||
__fish_is_running_tests="$homedir"
|
||||
export __fish_is_running_tests
|
||||
|
||||
# Set a marker to indicate whether colored output should be suppressed (used in `test_util.fish`)
|
||||
suppress_color=""
|
||||
if ! tty 0>&1 > /dev/null; then
|
||||
suppress_color="yes"
|
||||
fi
|
||||
export suppress_color
|
||||
|
||||
# Source test util functions at startup
|
||||
fish_init_cmd="${fish_init_cmd} && source ${TESTS_ROOT}/test_util.fish";
|
||||
|
||||
# Indicate that the fish panic handler shouldn't wait for input to prevent tests from hanging
|
||||
FISH_FAST_FAIL=1
|
||||
export FISH_FAST_FAIL
|
||||
|
||||
# Run the test script, but don't exec so we can clean up after it succeeds/fails. Each test is
|
||||
# launched directly within its TMPDIR, so that the fish tests themselves do not need to refer to
|
||||
# TMPDIR (to ensure their output as displayed in case of failure by littlecheck is reproducible).
|
||||
if test -n "${@}"; then
|
||||
(cd $TMPDIR && env HOME="$homedir" fish_test_helper="$homedir/fish_test_helper" "$fish" \
|
||||
--init-command "${fish_init_cmd}" "$fish_script" "${@}")
|
||||
else
|
||||
(cd $TMPDIR && env HOME="$homedir" fish_test_helper="$homedir/fish_test_helper" "$fish" \
|
||||
--init-command "${fish_init_cmd}" "$fish_script")
|
||||
fi
|
||||
test_status="$?"
|
||||
|
||||
rm -rf "$homedir"
|
||||
exit "$test_status"
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
# vim: set ts=4 sw=4 tw=100 et:
|
||||
|
||||
# This script sets up a clean environment for a script or executable to execute in/under. It creates
|
||||
# (and sets) $TMPDIR initialized to a clean & unique temporary directory, creates a new $HOME, sets
|
||||
# the relevant XDG_* directories to point to subdirectories of that $HOME, cleans up any potentially
|
||||
# problematic environment variables, executes the provided command, waits for it to exit, then
|
||||
# cleans up the newly created environment before bubbling up the exit code. $PWD is not changed.
|
||||
# If sourced instead of executed, sets up the environment as before but does not execute any payload
|
||||
# and does not destroy the newly created environment.
|
||||
|
||||
# macOS has really weird default IFS behavior that splits output in random places, and the trailing
|
||||
# backspace is to prevent \n from being gobbled up by the subshell output substitution.
|
||||
# Folks, this is why you should use fish!
|
||||
IFS="$(printf "\n\b")"
|
||||
# set -ex
|
||||
|
||||
die() {
|
||||
if test "$#" -ge 0; then
|
||||
printf "%s\n" "$@" 1>&2
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Set up a test environment to run the specified target under. We do not share environments
|
||||
# whatsoever between tests, so each test driver run sets up a new profile altogether.
|
||||
|
||||
# macOS 10.10 requires an explicit template for `mktemp` and will create the folder in the
|
||||
# current directory unless told otherwise. Linux isn't guaranteed to have $TMPDIR set.
|
||||
homedir="$(mktemp -d 2>/dev/null || mktemp -d "${TMPDIR}tmp.XXXXXXXXXX")"
|
||||
export HOME="$homedir"
|
||||
|
||||
XDG_DATA_HOME="$homedir/xdg_data_home"
|
||||
export XDG_DATA_HOME
|
||||
mkdir -p $XDG_DATA_HOME/fish || die
|
||||
|
||||
XDG_CONFIG_HOME="$homedir/xdg_config_home"
|
||||
export XDG_CONFIG_HOME
|
||||
mkdir -p $XDG_CONFIG_HOME/fish || die
|
||||
|
||||
XDG_RUNTIME_DIR="$homedir/xdg_runtime_dir"
|
||||
export XDG_RUNTIME_DIR
|
||||
mkdir -p $XDG_RUNTIME_DIR/fish || die
|
||||
chmod 700 "$XDG_RUNTIME_DIR"
|
||||
|
||||
XDG_CACHE_HOME="$homedir/xdg_cache_home"
|
||||
export XDG_CACHE_HOME
|
||||
mkdir -p $XDG_CACHE_HOME/fish || die
|
||||
|
||||
# Create a temp/scratch directory for tests to use, if they want (tests shouldn't write to a
|
||||
# shared temp folder).
|
||||
TMPDIR="$homedir/temp"
|
||||
mkdir ${TMPDIR}
|
||||
export TMPDIR
|
||||
|
||||
# Set locale information for consistent tests. Fish should work with a lot of locales but the
|
||||
# tests assume an english UTF-8 locale unless they explicitly override this default. We do not
|
||||
# want the users locale to affect the tests since they might, for example, change the wording of
|
||||
# logged messages.
|
||||
#
|
||||
# TODO: set LANG to en_US.UTF-8 so we test the locale message conversions (i.e., gettext).
|
||||
unset LANGUAGE
|
||||
# Remove "LC_" env vars from the test environment
|
||||
for key in $(env | grep -E "^LC_"| grep -oE "^[^=]+"); do
|
||||
unset "$key"
|
||||
done
|
||||
# Set the desired lang/locale tests are hard-coded against
|
||||
export LANG="C"
|
||||
export LC_CTYPE="en_US.UTF-8"
|
||||
|
||||
# These env vars should not be inherited from the user environment because they can affect the
|
||||
# behavior of the tests. So either remove them or set them to a known value.
|
||||
# See also tests/interactive.fish.
|
||||
export TERM=xterm
|
||||
unset COLORTERM
|
||||
unset KONSOLE_PROFILE_NAME
|
||||
unset KONSOLE_VERSION
|
||||
unset PANTHEON_TERMINAL_ID
|
||||
unset LC_TERMINAL
|
||||
unset LC_TERMINAL_VERSION
|
||||
unset TERM_PROGRAM
|
||||
unset TERM_PROGRAM_VERSION
|
||||
unset VTE_VERSION
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
# vim: set ts=4 sw=4 tw=100 et:
|
||||
# Utilities for the test runners
|
||||
|
||||
function die
|
||||
set -q argv[1]; and echo $argv[1] >&2
|
||||
exit 1
|
||||
end
|
||||
|
||||
# $suppress_color is set by `test_driver.sh` (via import of exported variables)
|
||||
function say -V suppress_color
|
||||
set -l color_flags
|
||||
set -l suppress_newline
|
||||
while set -q argv[1]
|
||||
switch $argv[1]
|
||||
case -b -o -u
|
||||
set color_flags $color_flags $argv[1]
|
||||
case -n
|
||||
set suppress_newline 1
|
||||
case --
|
||||
set -e argv[1]
|
||||
break
|
||||
case -\*
|
||||
continue
|
||||
case \*
|
||||
break
|
||||
end
|
||||
set -e argv[1]
|
||||
end
|
||||
|
||||
if not set -q argv[2]
|
||||
echo 'usage: say [flags] color string [string...]' >&2
|
||||
return 1
|
||||
end
|
||||
|
||||
if begin
|
||||
test -n "$suppress_color"; or set_color $color_flags $argv[1]
|
||||
end
|
||||
printf '%s' $argv[2..-1]
|
||||
test -z "$suppress_color"; and set_color normal
|
||||
if test -z "$suppress_newline"
|
||||
echo
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# lame timer
|
||||
for program in {g,}date
|
||||
if command -q $program && $program --version 1>/dev/null 2>/dev/null
|
||||
set -g milli $program
|
||||
set -g unit ms
|
||||
break
|
||||
else
|
||||
set -g unit sec
|
||||
end
|
||||
end
|
||||
|
||||
function timestamp
|
||||
set -q milli[1]
|
||||
and $milli +%s%3N
|
||||
or date +%s
|
||||
end
|
||||
|
||||
function delta
|
||||
set -q milli[1]
|
||||
and math "( "($milli +%s%3N)" - $argv[1])"
|
||||
or math (date +%s) - $argv[1]
|
||||
end
|
||||
Loading…
Reference in a new issue