diff --git a/cmake/Tests.cmake b/cmake/Tests.cmake index 87bec586d..00133da11 100644 --- a/cmake/Tests.cmake +++ b/cmake/Tests.cmake @@ -100,8 +100,8 @@ foreach(CHECK ${FISH_CHECKS}) get_filename_component(CHECK_NAME ${CHECK} NAME) get_filename_component(CHECK ${CHECK} NAME_WE) add_test(NAME ${CHECK_NAME} - COMMAND env FISHDIR=${CMAKE_CURRENT_BINARY_DIR}/ ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.sh - ${CMAKE_CURRENT_BINARY_DIR}/tests/test.fish ${CHECK} + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.py ${CMAKE_CURRENT_BINARY_DIR} + checks/${CHECK}.fish WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests ) set_tests_properties(${CHECK_NAME} PROPERTIES SKIP_RETURN_CODE ${SKIP_RETURN_CODE}) @@ -113,8 +113,8 @@ FILE(GLOB PEXPECTS CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/tests/pexpects/*.py) foreach(PEXPECT ${PEXPECTS}) get_filename_component(PEXPECT ${PEXPECT} NAME) add_test(NAME ${PEXPECT} - COMMAND env FISHDIR=${CMAKE_CURRENT_BINARY_DIR}/ ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.sh - ${CMAKE_CURRENT_BINARY_DIR}/tests/interactive.fish ${PEXPECT} + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.py ${CMAKE_CURRENT_BINARY_DIR} + pexpects/${PEXPECT} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests ) set_tests_properties(${PEXPECT} PROPERTIES SKIP_RETURN_CODE ${SKIP_RETURN_CODE}) diff --git a/tests/checks/command-not-found.fish b/tests/checks/command-not-found.fish index d3223d774..82b2a7beb 100644 --- a/tests/checks/command-not-found.fish +++ b/tests/checks/command-not-found.fish @@ -1,4 +1,4 @@ -#RUN: fish=%fish %fish %s | %fish %filter-control-sequences +#RUN: fish=%fish %fish %s set -g PATH $fish -c "nonexistent-command-1234 banana rama" #CHECKERR: fish: Unknown command: nonexistent-command-1234 diff --git a/tests/checks/complete_directories.fish b/tests/checks/complete_directories.fish index aa292f447..681a8c633 100644 --- a/tests/checks/complete_directories.fish +++ b/tests/checks/complete_directories.fish @@ -1,4 +1,4 @@ -#RUN: %fish --interactive %s | %fish %filter-control-sequences +#RUN: %fish --interactive %s # ^ interactive so we can do `complete` mkdir -p __fish_complete_directories/ cd __fish_complete_directories diff --git a/tests/checks/git.fish b/tests/checks/git.fish index 1fe07db4b..2dae920ed 100644 --- a/tests/checks/git.fish +++ b/tests/checks/git.fish @@ -1,4 +1,4 @@ -#RUN: %fish -i %s | %fish %filter-control-sequences +#RUN: %fish -i %s # Note: ^ this is interactive so we test interactive behavior, # e.g. the fish_git_prompt variable handlers test `status is-interactive`. #REQUIRES: command -v git diff --git a/tests/checks/invocation.fish b/tests/checks/invocation.fish index d602e1cc1..2c16b10cc 100644 --- a/tests/checks/invocation.fish +++ b/tests/checks/invocation.fish @@ -1,4 +1,4 @@ -#RUN: fish=%fish %fish %s | %fish %filter-control-sequences +#RUN: fish=%fish %fish %s $fish -c "echo 1.2.3.4." # CHECK: 1.2.3.4. diff --git a/tests/checks/read.fish b/tests/checks/read.fish index 845a80576..93f5a54a9 100644 --- a/tests/checks/read.fish +++ b/tests/checks/read.fish @@ -1,4 +1,4 @@ -# RUN: fish=%fish filter_ctrls=%filter-control-sequences %fish %s +# RUN: fish=%fish %fish %s # Set term again explicitly to ensure behavior. set -gx TERM xterm # Read with no vars is not an error @@ -248,7 +248,7 @@ if test (string length "$x") -ne $fish_read_limit end # Confirm reading non-interactively works -- \#4206 regression -echo abc\ndef | $fish -i -c 'read a; read b; set --show a; set --show b' | $fish $filter_ctrls +echo abc\ndef | $fish -i -c 'read a; read b; set --show a; set --show b' #CHECK: $a: set in global scope, unexported, with 1 elements #CHECK: $a[1]: |abc| #CHECK: $b: set in global scope, unexported, with 1 elements diff --git a/tests/checks/return.fish b/tests/checks/return.fish index 1cdcc3ec1..f1c19eceb 100644 --- a/tests/checks/return.fish +++ b/tests/checks/return.fish @@ -1,4 +1,4 @@ -#RUN: fish=%fish filter_ctrls=%filter-control-sequences %fish %s +#RUN: fish=%fish %fish %s # Some tests of the "return" builtin. $fish -c 'return 5' @@ -21,7 +21,7 @@ begin # but not bar echo $status # CHECK: 69 -end | $fish $filter_ctrls +end # Verify negative return values don't cause UB and never map to 0 function empty_return diff --git a/tests/checks/set.fish b/tests/checks/set.fish index d733dcfd1..1cb97ec3a 100644 --- a/tests/checks/set.fish +++ b/tests/checks/set.fish @@ -1,4 +1,4 @@ -# RUN: env FISH=%fish filter_ctrls=%filter-control-sequences %fish %s +# RUN: env FISH=%fish %fish %s # Environment variable tests # Test if variables can be properly set @@ -367,7 +367,7 @@ begin env SHLVL=" 3" $FISH -ic 'echo SHLVL: $SHLVL' # CHECK: SHLVL: 4 # CHECK: SHLVL: 4 -end | $FISH $filter_ctrls +end # Non-interactive fish doesn't touch $SHLVL env SHLVL=2 $FISH -c 'echo SHLVL: $SHLVL' diff --git a/tests/test_driver.py b/tests/test_driver.py new file mode 100755 index 000000000..f3cecbf0f --- /dev/null +++ b/tests/test_driver.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +import os +from datetime import datetime +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +import littlecheck + +try: + import pexpect + + PEXPECT = True +except ImportError: + PEXPECT = False + +RESET = "\033[0m" +GREEN = "\033[32m" +BLUE = "\033[34m" +RED = "\033[31m" + + +def main(): + if len(sys.argv) < 2: + print("Usage: test_driver.py FISH_DIRECTORY TESTS") + return 1 + + fishdir = Path(sys.argv[1]).absolute() + if not fishdir.is_dir(): + fishdir = fishdir.parent + script_path = Path(__file__).parent + + failcount = 0 + passcount = 0 + skipcount = 0 + def_subs = {"%": "%"} + lconfig = littlecheck.Config() + lconfig.colorize = sys.stdout.isatty() + lconfig.progress = True + + for bin in ["fish", "fish_indent", "fish_key_reader"]: + if os.path.exists(fishdir / bin): + def_subs[bin] = str(fishdir / bin) + else: + print(f"Binary does not exist: {bin}") + return 127 + + if len(sys.argv) > 2: + files = [(os.path.abspath(path), path) for path in sys.argv[2:]] + else: + files = [ + (os.path.abspath(path), path.relative_to(script_path)) + for path in script_path.glob("checks/*.fish") + ] + files += [ + (os.path.abspath(path), path.relative_to(script_path)) + for path in script_path.glob("pexpects/*.py") + ] + + # Set up tempdir + # "delete=" was added in 3.12. + home = tempfile.TemporaryDirectory(prefix="fishtest-") + xdg_config = home.name + "/xdg_config_home" + func_dir = xdg_config + "/fish/functions" + os.makedirs(func_dir) + os.makedirs(xdg_config + "/fish/conf.d/") + for func in (script_path / "test_functions").glob("*.fish"): + shutil.copy(func, func_dir + "/" + func.parts[-1]) + shutil.copy( + script_path / "interactive.config", xdg_config + "/fish/conf.d/interactive.fish" + ) + + xdg_data = home.name + "/xdg_data_home" + os.makedirs(xdg_data) + xdg_runtime = home.name + "/xdg_runtime_home" + os.makedirs(xdg_runtime) + xdg_cache = home.name + "/xdg_cache_home" + os.makedirs(xdg_cache) + tmp = home.name + "/temp" + os.makedirs(tmp) + + # Compile fish_test_helper if necessary. + # If we're run multiple times, keep this around to save time. + # TODO: It's cheesy to leave this in the current dir + if not os.path.exists("fish_test_helper"): + comp = subprocess.run( + ["cc", script_path / "fish_test_helper.c", "-o", "fish_test_helper"] + ) + shutil.copy("fish_test_helper", home.name + "/fish_test_helper") + def_subs.update({"fish_test_helper": home.name + "/fish_test_helper"}) + + # unset LANG, TERM, ... + for var in [ + "XDG_DATA_DIRS", + "LANGUAGE", + "COLORTERM", + "KONSOLE_PROFILE_NAME", + "KONSOLE_VERSION", + "TERM_PROGRAM", + "TERM_PROGRAM_VERSION", + "VTE_VERSION", + ]: + if var in os.environ: + del os.environ[var] + langvars = [key for key in os.environ.keys() if key.startswith("LC_")] + for key in langvars: + del os.environ[key] + + os.environ.update( + { + "HOME": home.name, + "TMPDIR": tmp, + "FISH_FAST_FAIL": "1", + "FISH_UNIT_TESTS_RUNNING": "1", + "XDG_CONFIG_HOME": xdg_config, + "XDG_DATA_HOME": xdg_data, + "XDG_RUNTIME_DIR": xdg_runtime, + "XDG_CACHE_HOME": xdg_cache, + "fish_test_helper": home.name + "/fish_test_helper", + "TERM": "xterm", + "LANG": "C", + "LC_CTYPE": "en_US.UTF-8", + } + ) + + # environ for py files has a few changes. + pyenviron = os.environ.copy() + pyenviron.update( + { + "PYTHONPATH": str(script_path), + "fish": str(fishdir / "fish"), + "fish_key_reader": str(fishdir / "fish_key_reader"), + "fish_indent": str(fishdir / "fish_indent"), + "TERM": "dumb", + "FISH_FORCE_COLOR": "1" if sys.stdout.isatty() else "0", + } + ) + + os.chdir(home.name) + + print(f"Checking files (TMPDIR is {home.name})") + if not PEXPECT and any(x.endswith(".py") for (x, _) in files): + print(f"{RED}Skipping pexpect tests because pexpect is not installed{RESET}") + + for f, arg in files: + starttime = datetime.now() + if f.endswith(".fish"): + subs = def_subs.copy() + subs["s"] = f + # littlecheck + print(f"{arg}..", end="", flush=True) + ret = littlecheck.check_path(f, subs, lconfig, lambda x: print(x.message())) + endtime = datetime.now() + duration_ms = round((endtime - starttime).total_seconds() * 1000) + if ret is littlecheck.SKIP: + print(f"{BLUE}SKIPPED{RESET}") + skipcount += 1 + elif ret: + print(f"{GREEN}PASS{RESET} ({duration_ms} ms)") + passcount += 1 + else: + print(f"{RED}FAIL{RESET} ({duration_ms} ms)") + failcount += 1 + elif f.endswith(".py"): + print(f"{arg}..", end="", flush=True) + if not PEXPECT: + print(f"{BLUE}SKIPPED{RESET}") + skipcount += 1 + continue + proc = subprocess.run( + ["python3", f], + capture_output=True, + env=pyenviron, + ) + endtime = datetime.now() + duration_ms = round((endtime - starttime).total_seconds() * 1000) + if proc.returncode == 0: + print(f"{GREEN}PASS{RESET} ({duration_ms} ms)") + passcount += 1 + elif proc.returncode == 127: + print(f"{BLUE}SKIPPED{RESET}") + skipcount += 1 + else: + print(f"{RED}FAILED{RESET} ({duration_ms} ms)") + if proc.stdout: + print(proc.stdout.decode("utf-8")) + if proc.stderr: + print(proc.stderr.decode("utf-8")) + failcount += 1 + else: + print(f"Not a valid test file: {arg}") + failcount += 1 + if passcount + failcount + skipcount > 1: + print(f"{passcount} / {passcount + failcount} passed ({skipcount} skipped)") + if passcount == 0 and failcount == 0 and skipcount: + return 125 + return 1 if failcount else 0 + + +if __name__ == "__main__": + try: + ret = main() + sys.exit(ret) + except KeyboardInterrupt: + sys.exit(130)