mirror of
https://github.com/fish-shell/fish-shell
synced 2024-11-10 07:04:29 +00:00
Run each test fully independently in own environment
This commit is contained in:
Mahmoud Al-Qudsi
Johannes Altmanninger
parent
498e5fa9b0
commit
35975d83af
5 changed files with 166 additions and 156 deletions
|
|
@ -175,7 +175,7 @@ install(FILES fish.png DESTINATION ${rel_datadir}/pixmaps)
|
|||
|
||||
# Group install targets into a InstallTargets folder
|
||||
set_property(TARGET build_fish_pc CHECK-FISH-BUILD-VERSION-FILE
|
||||
test_fishscript
|
||||
# test_fishscript
|
||||
test_prep tests_buildroot_target
|
||||
PROPERTY FOLDER cmake/InstallTargets)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,14 @@
|
|||
# This adds ctest support to the project
|
||||
enable_testing()
|
||||
|
||||
# By default, ctest runs tests serially
|
||||
if(NOT CTEST_PARALLEL_LEVEL)
|
||||
include(ProcessorCount)
|
||||
ProcessorCount(CORES)
|
||||
message("CTEST_PARALLEL_LEVEL ${CORES}")
|
||||
set(CTEST_PARALLEL_LEVEL ${CORES})
|
||||
endif()
|
||||
|
||||
# Define fish_tests.
|
||||
add_executable(fish_tests EXCLUDE_FROM_ALL
|
||||
src/fish_tests.cpp)
|
||||
|
|
@ -6,6 +17,25 @@ fish_link_deps_and_sign(fish_tests)
|
|||
# The "test" directory.
|
||||
set(TEST_DIR ${CMAKE_CURRENT_BINARY_DIR}/test)
|
||||
|
||||
# HACK: CMake is a configuration tool, not a build system. However, until CMake adds a way to
|
||||
# dynamically discover tests, our options are either this or resorting to sed/awk to parse the low
|
||||
# level tests source file to get the list of individual tests. Or to split each test into its own
|
||||
# source file.
|
||||
execute_process(
|
||||
COMMAND ${CMAKE_CXX_COMPILER} ${CMAKE_SOURCE_DIR}/src/fish_tests.cpp
|
||||
-o ${CMAKE_BINARY_DIR}/fish_tests_list
|
||||
-I ${CMAKE_CURRENT_BINARY_DIR} # for config.h
|
||||
-lpthread
|
||||
# Strip actual dependency on fish code
|
||||
-Wl,-undefined,dynamic_lookup,--unresolved-symbols=ignore-all
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
|
||||
)
|
||||
execute_process(
|
||||
COMMAND ./fish_tests_list --list
|
||||
OUTPUT_FILE low_level_tests.txt
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
|
||||
)
|
||||
|
||||
# The directory into which fish is installed.
|
||||
set(TEST_INSTALL_DIR ${TEST_DIR}/buildroot)
|
||||
|
||||
|
|
@ -77,74 +107,25 @@ add_custom_target(test_prep
|
|||
DEPENDS tests_buildroot_target tests_dir
|
||||
USES_TERMINAL)
|
||||
|
||||
# Define our individual tests.
|
||||
# Each test is conceptually independent.
|
||||
# However when running all tests, we want to run them serially for sanity's sake.
|
||||
# So define both a normal target, and a serial variant which enforces ordering.
|
||||
foreach(TESTTYPE test serial_test)
|
||||
add_custom_target(${TESTTYPE}_low_level
|
||||
COMMAND env XDG_DATA_DIRS=
|
||||
XDG_DATA_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_data
|
||||
XDG_CONFIG_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_config
|
||||
XDG_RUNTIME_DIR=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_runtime
|
||||
./fish_tests
|
||||
foreach(LTEST ${LOW_LEVEL_TESTS})
|
||||
add_test(
|
||||
NAME ${LTEST}
|
||||
COMMAND ${CMAKE_BINARY_DIR}/fish_tests ${LTEST}
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
|
||||
DEPENDS fish_tests
|
||||
USES_TERMINAL)
|
||||
)
|
||||
endforeach(LTEST)
|
||||
|
||||
add_custom_target(${TESTTYPE}_fishscript
|
||||
COMMAND
|
||||
cd tests &&
|
||||
env XDG_DATA_DIRS=
|
||||
XDG_DATA_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_data
|
||||
XDG_CONFIG_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_config
|
||||
XDG_RUNTIME_DIR=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_runtime
|
||||
${TEST_ROOT_DIR}/bin/fish test.fish
|
||||
DEPENDS test_prep
|
||||
USES_TERMINAL)
|
||||
add_test(test_prep
|
||||
"${CMAKE_COMMAND}" --build ${CMAKE_BINARY_DIR} --target test_prep)
|
||||
FILE(GLOB FISH_CHECKS CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/tests/checks/*.fish)
|
||||
foreach(CHECK ${FISH_CHECKS})
|
||||
get_filename_component(CHECK_NAME ${CHECK} NAME)
|
||||
get_filename_component(CHECK ${CHECK} NAME_WE)
|
||||
add_test(NAME ${CHECK_NAME}
|
||||
COMMAND sh ${CMAKE_CURRENT_BINARY_DIR}/tests/test_driver.sh
|
||||
${CMAKE_CURRENT_BINARY_DIR}/tests/test.fish ${CHECK}
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests
|
||||
)
|
||||
|
||||
add_custom_target(${TESTTYPE}_interactive
|
||||
COMMAND cd tests &&
|
||||
env XDG_DATA_DIRS=
|
||||
XDG_DATA_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_data
|
||||
XDG_CONFIG_HOME=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_config
|
||||
XDG_RUNTIME_DIR=${CMAKE_CURRENT_BINARY_DIR}/test/xdg_runtime
|
||||
${TEST_ROOT_DIR}/bin/fish interactive.fish
|
||||
DEPENDS test_prep
|
||||
USES_TERMINAL)
|
||||
endforeach(TESTTYPE)
|
||||
|
||||
# Now add a dependency chain between the serial versions.
|
||||
# This ensures they run in order.
|
||||
add_dependencies(serial_test_low_level test_prep)
|
||||
add_dependencies(serial_test_fishscript serial_test_low_level)
|
||||
add_dependencies(serial_test_interactive serial_test_fishscript)
|
||||
|
||||
|
||||
add_custom_target(serial_test_high_level
|
||||
DEPENDS serial_test_interactive serial_test_fishscript)
|
||||
|
||||
# Create the 'test' target.
|
||||
# Set a policy so CMake stops complaining about the name 'test'.
|
||||
cmake_policy(PUSH)
|
||||
|
||||
if(${CMAKE_VERSION} VERSION_LESS 3.11.0 AND POLICY CMP0037)
|
||||
cmake_policy(SET CMP0037 OLD)
|
||||
endif()
|
||||
add_custom_target(test)
|
||||
cmake_policy(POP)
|
||||
add_dependencies(test serial_test_high_level)
|
||||
|
||||
# Group test targets into a TestTargets folder
|
||||
set_property(TARGET test tests_dir
|
||||
test_low_level
|
||||
test_fishscript
|
||||
test_interactive
|
||||
test_fishscript test_prep
|
||||
tests_buildroot_target
|
||||
serial_test_high_level
|
||||
serial_test_low_level
|
||||
serial_test_fishscript
|
||||
serial_test_interactive
|
||||
symlink_functions
|
||||
PROPERTY FOLDER cmake/TestTargets)
|
||||
set_tests_properties(${LTEST} PROPERTIES DEPENDS test_prep)
|
||||
endforeach(CHECK)
|
||||
|
|
|
|||
|
|
@ -19,10 +19,6 @@ else
|
|||
set files_to_test checks/*.fish
|
||||
end
|
||||
|
||||
# test_util handles the environment setup and then restarts us
|
||||
source test_util.fish (status -f) $argv
|
||||
or exit
|
||||
|
||||
say -o cyan "Testing high level script functionality"
|
||||
|
||||
set -g python (__fish_anypython)
|
||||
|
|
|
|||
115
tests/test_driver.sh
Normal file
115
tests/test_driver.sh
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
# vim: set ts=4 sw=4 tw=100 et:
|
||||
# POSIX sh test driver to reduce dependency on fish in tests
|
||||
|
||||
# macOS has really weird default IFS behavior that splits output in random places, and the trailing
|
||||
# backspace is to prevent \n from being gobbled up by the subshell output substitution.
|
||||
# Folks, this is why you should use fish!
|
||||
IFS="$(printf "\n\b")"
|
||||
|
||||
# The first argument is the path to the script to launch; all remaining arguments are forwarded to
|
||||
# the script.
|
||||
fish_script="$1"
|
||||
shift 1
|
||||
script_args="${@}"
|
||||
|
||||
die() {
|
||||
if test "$#" -ge 0; then
|
||||
printf "%s\n" "$@" 1>&2
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
# To keep things sane and to make error messages comprehensible, do not use relative paths anywhere
|
||||
# in this script. Instead, make all paths relative to one of these or $homedir."
|
||||
TESTS_ROOT="$(realpath "$(dirname "$0")")"
|
||||
BUILD_ROOT="$(realpath "${TESTS_ROOT}/..")"
|
||||
|
||||
if ! test -z "$__fish_is_running_tests"; then
|
||||
echo "Recursive test invocation detected!" 1>&2
|
||||
exit 10
|
||||
fi
|
||||
|
||||
# Set up a test environment and re-run the original script. We do not share environments
|
||||
# whatsoever between tests, so each test driver run sets up a new profile altogether.
|
||||
|
||||
homedir="$(mktemp -d)"
|
||||
|
||||
# cp -a ../test/xdg_data_home "$homedir/"
|
||||
XDG_DATA_HOME="$homedir/xdg_data_home"
|
||||
export XDG_DATA_HOME
|
||||
mkdir -p $XDG_DATA_HOME/fish || die
|
||||
|
||||
# cp -a ../test/xdg_config_home "$homedir/"
|
||||
XDG_CONFIG_HOME="$homedir/xdg_config_home"
|
||||
export XDG_CONFIG_HOME
|
||||
mkdir -p $XDG_CONFIG_HOME/fish || die
|
||||
|
||||
XDG_RUNTIME_DIR="$homedir/xdg_runtime_dir"
|
||||
export XDG_CONFIG_HOME
|
||||
mkdir -p $XDG_RUNTIME_DIR/fish || die
|
||||
|
||||
# These are used read-only so it's OK to symlink instead of copy
|
||||
rm -f "$XDG_CONFIG_HOME/fish/functions"
|
||||
ln -s "$PWD/test_functions" "$XDG_CONFIG_HOME/fish/functions" || die "Failed to symlink"
|
||||
|
||||
# Set the function path at startup, referencing the default fish functions and the test-specific
|
||||
# functions.
|
||||
fish_init_cmd="set fish_function_path ${XDG_CONFIG_HOME}/fish/functions ${BUILD_ROOT}/share/functions"
|
||||
|
||||
__fish_is_running_tests="$homedir"
|
||||
export __fish_is_running_tests
|
||||
|
||||
# Set locale information for consistent tests. Fish should work with a lot of locales but the
|
||||
# tests assume an english UTF-8 locale unless they explicitly override this default. We do not
|
||||
# want the users locale to affect the tests since they might, for example, change the wording of
|
||||
# logged messages.
|
||||
#
|
||||
# TODO: set LANG to en_US.UTF-8 so we test the locale message conversions (i.e., gettext).
|
||||
unset LANGUAGE
|
||||
# Remove "LC_" env vars from the test environment
|
||||
for key in $(env | grep -E "^LC_"| grep -oE "^[^=]+"); do
|
||||
unset "$key"
|
||||
done
|
||||
# Set the desired lang/locale tests are hard-coded against
|
||||
export LANG="C"
|
||||
export LC_CTYPE="en_US.UTF-8"
|
||||
|
||||
# These env vars should not be inherited from the user environment because they can affect the
|
||||
# behavior of the tests. So either remove them or set them to a known value.
|
||||
# See also tests/interactive.fish.
|
||||
export TERM=xterm
|
||||
unset COLORTERM
|
||||
unset INSIDE_EMACS
|
||||
unset ITERM_PROFILE
|
||||
unset KONSOLE_PROFILE_NAME
|
||||
unset KONSOLE_VERSION
|
||||
unset PANTHEON_TERMINAL_ID
|
||||
unset TERM_PROGRAM
|
||||
unset TERM_PROGRAM_VERSION
|
||||
unset VTE_VERSION
|
||||
unset WT_PROFILE_ID
|
||||
unset XTERM_VERSION
|
||||
|
||||
# Set a marker to indicate whether colored output should be suppressed (used in `test_util.fish`)
|
||||
suppress_color=""
|
||||
if ! tty 0>&1 > /dev/null; then
|
||||
suppress_color="yes"
|
||||
fi
|
||||
export suppress_color
|
||||
|
||||
# Source test util functions at startup
|
||||
fish_init_cmd="${fish_init_cmd} && source ${TESTS_ROOT}/test_util.fish";
|
||||
|
||||
# Run the test script, but don't exec so we can do cleanup after it succeeds/fails
|
||||
# echo $PWD
|
||||
# echo "BUILD_ROOT: $BUILD_ROOT"
|
||||
# echo "TESTS_ROOT: $TESTS_ROOT"
|
||||
env HOME=$homedir "${BUILD_ROOT}/test/root/bin/fish" \
|
||||
--init-command "${fish_init_cmd}" \
|
||||
"$fish_script" "$script_args"
|
||||
test_status="$?"
|
||||
|
||||
echo test completed in $homedir with status $test_status
|
||||
|
||||
rm -rf "$homedir"
|
||||
exit "$test_status"
|
||||
|
|
@ -1,94 +1,12 @@
|
|||
# vim: set ts=4 sw=4 tw=100 et:
|
||||
# Utilities for the test runners
|
||||
|
||||
if test "$argv[1]" = (status -f)
|
||||
echo 'test_util.fish requires sourcing script as argument to `source`' >&2
|
||||
echo 'use `source test_util.fish (status -f); or exit`' >&2
|
||||
status --print-stack-trace >&2
|
||||
exit 1
|
||||
end
|
||||
|
||||
# Any remaining arguments are passed back to test.fish
|
||||
set -l args_for_test_script
|
||||
if set -q argv[2]
|
||||
set args_for_test_script $argv[2..-1]
|
||||
end
|
||||
|
||||
function die
|
||||
set -q argv[1]; and echo $argv[1] >&2
|
||||
exit 1
|
||||
end
|
||||
|
||||
# Check if we're running in the test environment. If not, set it up and rerun fish with exec. The
|
||||
# test is whether the special var __fish_is_running_tests exists and contains the same value as
|
||||
# XDG_CONFIG_HOME. It checks the value and not just the presence because we're going to delete the
|
||||
# config directory later if we're exiting successfully.
|
||||
if not set -q __fish_is_running_tests
|
||||
# Set up our test environment and re-run the original script.
|
||||
set -l script $argv[1]
|
||||
|
||||
cd (builtin realpath (dirname $script))
|
||||
or die
|
||||
|
||||
set -lx XDG_DATA_HOME ../test/xdg_data_home
|
||||
rm -rf $XDG_DATA_HOME/fish
|
||||
mkdir -p $XDG_DATA_HOME/fish; or die
|
||||
|
||||
set -lx XDG_CONFIG_HOME ../test/xdg_config_home
|
||||
rm -rf $XDG_CONFIG_HOME/fish
|
||||
mkdir -p $XDG_CONFIG_HOME/fish; or die
|
||||
ln -s $PWD/test_functions $XDG_CONFIG_HOME/fish/functions; or die
|
||||
|
||||
set -l escaped_parent (builtin realpath $PWD/.. | string escape); or die
|
||||
set -l escaped_config (string escape -- $XDG_CONFIG_HOME/fish)
|
||||
printf 'set fish_function_path \'%s/functions\' \'%s/share/functions\'\n' $escaped_config $escaped_parent >$XDG_CONFIG_HOME/fish/config.fish; or die
|
||||
set -xl __fish_is_running_tests $XDG_CONFIG_HOME
|
||||
|
||||
# Set locale information for consistent tests. Fish should work with a lot of locales but the
|
||||
# tests assume an english UTF-8 locale unless they explicitly override this default. We do not
|
||||
# want the users locale to affect the tests since they might, for example, change the wording of
|
||||
# logged messages.
|
||||
#
|
||||
# TODO: set LANG to en_US.UTF-8 so we test the locale message conversions (i.e., gettext).
|
||||
set -e LANGUAGE
|
||||
set -x LANG C
|
||||
# Remove "LC_" env vars from the test environment.
|
||||
for var in (set -xn)
|
||||
string match -q 'LC_*' $var
|
||||
and set -e $var
|
||||
end
|
||||
set -x LC_CTYPE en_US.UTF-8
|
||||
|
||||
# These env vars should not be inherited from the user environment because they can affect the
|
||||
# behavior of the tests. So either remove them or set them to a known value.
|
||||
# See also tests/interactive.fish.
|
||||
set -gx TERM xterm
|
||||
set -e COLORTERM
|
||||
set -e INSIDE_EMACS
|
||||
set -e ITERM_PROFILE
|
||||
set -e KONSOLE_PROFILE_NAME
|
||||
set -e KONSOLE_VERSION
|
||||
set -e PANTHEON_TERMINAL_ID
|
||||
set -e TERM_PROGRAM
|
||||
set -e TERM_PROGRAM_VERSION
|
||||
set -e VTE_VERSION
|
||||
set -e WT_PROFILE_ID
|
||||
set -e XTERM_VERSION
|
||||
|
||||
exec ../test/root/bin/fish $script $args_for_test_script
|
||||
die 'exec failed'
|
||||
else if test "$__fish_is_running_tests" != "$XDG_CONFIG_HOME"
|
||||
echo 'Something went wrong with the test runner.' >&2
|
||||
echo "__fish_is_running_tests: $__fish_is_running_tests" >&2
|
||||
echo "XDG_CONFIG_HOME: $XDG_CONFIG_HOME" >&2
|
||||
exit 10
|
||||
end
|
||||
|
||||
set -l suppress_color
|
||||
if not tty 0>&1 >/dev/null
|
||||
set suppress_color yes
|
||||
end
|
||||
|
||||
# $suppress_color is set by `test_driver.sh` (via import of exported variables)
|
||||
function say -V suppress_color
|
||||
set -l color_flags
|
||||
set -l suppress_newline
|
||||
|
|
|
|||
Loading…
Reference in a new issue