Add pexpect-based interactive testing framework

This adds a new interactive test framework based on Python's pexpect. This
is intended to supplant the TCL expect-based tests.

New tests go in `tests/pexpects/`. As a proof-of-concept, the
pipeline.expect test and the (gnarly) bind.expect test are ported to the
new framework.
This commit is contained in:
ridiculousfish 2020-03-02 15:20:29 -08:00
parent 218fe15264
commit 3b7feb38e9
5 changed files with 613 additions and 12 deletions

View file

@ -0,0 +1,261 @@
"""pexpect_helper provides a wrapper around the pexpect module.
This module exposes a single class SpawnedProc, which wraps pexpect.spawn().
This exposes a pseudo-tty, which fish or another process may talk to.
The send() function may be used to send data to fish, and the expect_* family
of functions may be used to match what is output to the tty.
Example usage:
sp = SpawnedProc() # this launches fish
sp.expect_prompt() # wait for a prompt
sp.sendline("echo hello world")
sp.expect_prompt("hello world")
"""
from __future__ import print_function
import inspect
import os
import os.path
import re
import sys
import time
import pexpect
# Default timeout for failing to match.
TIMEOUT_SECS = 5
def get_prompt_re(counter):
""" Return a regular expression for matching a with a given prompt counter. """
return re.compile(
r"""(?:\r\n?|^) # beginning of line
(?:\[.\]\ )? # optional vi mode prompt
"""
+ (r"prompt\ %d>" % counter), # prompt with counter
re.VERBOSE,
)
def get_callsite():
""" Return a triple (filename, line_number, line_text) of the call site location. """
callstack = inspect.getouterframes(inspect.currentframe())
for f in callstack:
if inspect.getmodule(f.frame) is not Message.MODULE:
return (os.path.basename(f.filename), f.lineno, f.code_context)
return ("Unknown", -1, "")
def escape(s):
""" Escape the string 's' to make it human-understandable. """
res = []
for c in s:
if c == "\n":
res.append("\\n")
elif c == "\r":
res.append("\\r")
elif c == "\t":
res.append("\\t")
elif c.isprintable():
res.append(c)
else:
res.append("\\x{:02x}".format(ord(c)))
return "".join(res)
class Message(object):
""" Some text either sent-to or received-from the spawned proc.
Attributes:
dir: the message direction, either DIR_SEND or DIR_RECV
filename: the name of the file from which the message was sent
text: the text of the messages
when: a timestamp of when the message was sent
"""
DIR_SEND = "SENT"
DIR_RECV = "RECV"
MODULE = sys.modules[__name__]
def __init__(self, dir, text, when):
""" Construct from a direction, message text and timestamp. """
self.dir = dir
self.filename, self.lineno, _ = get_callsite()
self.text = text
self.when = when
@staticmethod
def sent(text, when):
""" Return a SEND message with the given text. """
return Message(Message.DIR_SEND, text, when)
@staticmethod
def received(text, when):
""" Return a RECV message with the given text. """
return Message(Message.DIR_RECV, text, when)
def formatted(self):
""" Return a human-readable string representing this message. """
etext = escape(self.text)
timestamp = self.when * 1000.0
return "{dir} {timestamp:.2f} ({filename}:{lineno}): {etext}".format(
timestamp=timestamp, etext=etext, **vars(self)
)
class SpawnedProc(object):
""" A process, talking to our ptty. This wraps pexpect.spawn.
Attributes:
colorize: whether error messages should have ANSI color escapes
messages: list of Message sent and received, in-order
start_time: the timestamp of the first message, or None if none yet
spawn: the pexpect.spawn value
prompt_counter: the index of the prompt. This cooperates with the fish_prompt
function to ensure that each printed prompt is distinct.
"""
def __init__(self, name="fish", timeout=TIMEOUT_SECS, env=os.environ.copy()):
""" Construct from a name, timeout, and environment.
Args:
name: the name of the executable to launch, as a key into the
environment dictionary. By default this is 'fish' but may be
other executables.
timeout: A timeout to pass to pexpect. This indicates how long to wait
before giving up on some expected output.
env: a string->string dictionary, describing the environment variables.
"""
if name not in env:
raise ValueError("'name' variable not found in environment" % name)
exe_path = env.get(name)
self.colorize = sys.stdout.isatty()
self.messages = []
self.start_time = None
self.spawn = pexpect.spawn(exe_path, env=env, encoding="utf-8", timeout=timeout)
self.spawn.delaybeforesend = None
self.prompt_counter = 1
def time_since_first_message(self):
""" Return a delta in seconds since the first message, or 0 if this is the first. """
now = time.monotonic()
if not self.start_time:
self.start_time = now
return now - self.start_time
def send(self, s):
""" Cover over pexpect.spawn.send().
Send the given string to the tty, returning the number of bytes written.
"""
res = self.spawn.send(s)
when = self.time_since_first_message()
self.messages.append(Message.sent(s, when))
return res
def sendline(self, s):
""" Cover over pexpect.spawn.sendline().
Send the given string + linesep to the tty, returning the number of bytes written.
"""
return self.send(s + os.linesep)
def expect_re(self, pat, pat_desc=None, unmatched=None, **kwargs):
""" Cover over pexpect.spawn.expect().
Look through the "new" output of self.spawn until the given pattern is matched.
The pattern is typically a regular expression in string form, but may also be
any of the types accepted by pexpect.spawn.expect().
If the 'unmatched' parameter is given,
On failure, this prints an error and exits.
"""
try:
res = self.spawn.expect(pat, **kwargs)
when = self.time_since_first_message()
self.messages.append(Message.received(self.spawn.match.group(), when))
return res
except pexpect.ExceptionPexpect as err:
if not pat_desc:
pat_desc = str(pat)
self.report_exception_and_exit(pat_desc, unmatched, err)
def expect_str(self, s, **kwargs):
""" Cover over expect_re() which accepts a literal string. """
return self.expect_re(re.escape(s), **kwargs)
def expect_prompt(self, *args, **kwargs):
""" Convenience function which matches some text and then a prompt.
Match the given positional arguments as expect_re, and then look
for a prompt, bumping the prompt counter.
Returns None on success, and exits on failure.
Example:
sp.sendline("echo hello world")
sp.expect_prompt("hello world")
"""
if args:
self.expect_re(*args, **kwargs)
self.expect_re(
get_prompt_re(self.prompt_counter),
pat_desc="prompt %d" % self.prompt_counter,
)
self.prompt_counter += 1
def report_exception_and_exit(self, pat, unmatched, err):
""" Things have gone badly.
We have an exception 'err', some pexpect.ExceptionPexpect.
Report it to stdout, along with the offending call site.
If 'unmatched' is set, print it to stdout.
"""
colors = self.colors()
if unmatched:
print("{BOLD}{unmatched}{RESET}".format(unmatched=unmatched, **colors))
if isinstance(err, pexpect.EOF):
msg = "EOF"
elif isinstance(err, pexpect.TIMEOUT):
msg = "TIMEOUT"
else:
msg = "UNKNOWN"
filename, lineno, code_context = get_callsite()
print("{RED}Failed to match:{NORMAL} {pat}".format(pat=escape(pat), **colors))
print(
"{msg} from {filename}:{lineno}: {code}".format(
msg=msg, filename=filename, lineno=lineno, code="\n".join(code_context)
)
)
# Show the last 5 messages.
for m in self.messages[-5:]:
print(m.formatted())
print("Buffer:")
print(escape(self.spawn.before))
sys.exit(1)
def sleep(self, secs):
""" Cover over time.sleep(). """
time.sleep(secs)
def colors(self):
""" Return a dictionary mapping color names to ANSI escapes """
def ansic(n):
""" Return either an ANSI escape sequence for a color, or empty string. """
return "\033[%dm" % n if self.colorize else ""
return {
"RESET": ansic(0),
"BOLD": ansic(1),
"NORMAL": ansic(39),
"BLACK": ansic(30),
"RED": ansic(31),
"GREEN": ansic(32),
"YELLOW": ansic(33),
"BLUE": ansic(34),
"MAGENTA": ansic(35),
"CYAN": ansic(36),
"LIGHTGRAY": ansic(37),
"DARKGRAY": ansic(90),
"LIGHTRED": ansic(91),
"LIGHTGREEN": ansic(92),
"LIGHTYELLOW": ansic(93),
"LIGHTBLUE": ansic(94),
"LIGHTMAGENTA": ansic(95),
"LIGHTCYAN": ansic(96),
"WHITE": ansic(97),
}

View file

@ -29,6 +29,9 @@ endif()
# Copy littlecheck.py
configure_file(build_tools/littlecheck.py littlecheck.py COPYONLY)
# Copy pexpect_helper.py
configure_file(build_tools/pexpect_helper.py pexpect_helper.py COPYONLY)
# Make the directory in which to run tests.
# Also symlink fish to where the tests expect it to be.
# Lastly put fish_test_helper there too.

View file

@ -22,11 +22,13 @@ cd (dirname (status -f))
set -gx TERM xterm
set -e ITERM_PROFILE
# Test files specified on commandline, or all *.expect files
# Test files specified on commandline, or all *.expect files.
if set -q argv[1]
set files_to_test $argv.expect
set expect_files_to_test $argv.expect
set pexpect_files_to_test pexpects/$argv.py
else
set files_to_test *.expect
set expect_files_to_test *.expect
set pexpect_files_to_test pexpects/*.py
end
source test_util.fish (status -f) $argv
@ -34,12 +36,7 @@ or exit
cat interactive.config >>$XDG_CONFIG_HOME/fish/config.fish
say -o cyan "Testing interactive functionality"
if not type -q expect
say red "Tests disabled: `expect` not found"
exit 0
end
function test_file
function test_expect_file
set -l file $argv[1]
echo -n "Testing file $file ... "
set starttime (timestamp)
@ -86,12 +83,53 @@ function test_file
end
end
function test_pexpect_file
set -l file $argv[1]
echo -n "Testing file $file ... "
begin
set starttime (timestamp)
set -lx TERM dumb
# Help the script find the pexpect_helper module in our parent directory.
set -lx --prepend PYTHONPATH (realpath $PWD/..)
set -lx fish ../test/root/bin/fish
set -lx fish_key_reader ../test/root/bin/fish_key_reader
set -lx fish_test_helper ../test/root/bin/fish_test_helper
# Note we require Python3.
python3 $file
end
set -l exit_status $status
if test "$exit_status" -eq 0
set test_duration (delta $starttime)
say green "ok ($test_duration $unit)"
end
return $exit_status
end
set failed
for i in $files_to_test
if not test_file $i
if not python3 -c 'import pexpect'
say red "pexpect tests disabled: `python3 -c 'import pexpect'` failed"
set pexpect_files_to_test
end
for i in $pexpect_files_to_test
if not test_pexpect_file $i
set failed $failed $i
end
end
if not type -q expect
say red "expect tests disabled: `expect` not found"
set expect_files_to_test
end
for i in $expect_files_to_test
if not test_expect_file $i
say -o cyan "Rerunning test $i"
rm -f $i.tmp.*
if not test_file $i
if not test_expect_file $i
set failed $failed $i
end
end

271
tests/pexpects/bind.py Executable file
View file

@ -0,0 +1,271 @@
#!/usr/bin/env python3
from pexpect_helper import SpawnedProc
sp = SpawnedProc()
sp.expect_prompt()
# Fish should start in default-mode (i.e., emacs) bindings. The default escape
# timeout is 30ms.
# Verify the emacs transpose word (\et) behavior using various delays,
# including none, after the escape character.
# Start by testing with no delay. This should transpose the words.
sp.send("echo abc def")
sp.send("\033t\r")
sp.expect_prompt("\r\ndef abc\r\n") # emacs transpose words, default timeout: no delay
# Now test with a delay > 0 and < the escape timeout. This should transpose
# the words.
sp.send("echo ghi jkl")
sp.send("\033")
sp.sleep(0.010)
sp.send("t\r")
# emacs transpose words, default timeout: short delay
sp.expect_prompt("\r\njkl ghi\r\n")
# Now test with a delay > the escape timeout. The transposition should not
# occur and the "t" should become part of the text that is echoed.
sp.send("echo mno pqr")
sp.send("\033")
sp.sleep(0.200)
sp.send("t\r")
# emacs transpose words, default timeout: long delay
sp.expect_prompt("\r\nmno pqrt\r\n")
# Now test that exactly the expected bind modes are defined
sp.sendline("bind --list-modes")
sp.expect_prompt("\r\ndefault\r\npaste", unmatched="Unexpected bind modes")
# Test vi key bindings.
# This should leave vi mode in the insert state.
sp.sendline("set -g fish_key_bindings fish_vi_key_bindings")
sp.expect_prompt()
# Go through a prompt cycle to let fish catch up, it may be slow due to ASAN
sp.sendline("echo success: default escape timeout")
sp.expect_prompt(
"\r\nsuccess: default escape timeout", unmatched="prime vi mode, default timeout"
)
sp.send("echo fail: default escape timeout")
sp.send("\033")
# Delay needed to allow fish to transition to vi "normal" mode. The delay is
# longer than strictly necessary to let fish catch up as it may be slow due to
# ASAN.
sp.sleep(0.150)
sp.send("ddi")
sp.sendline("echo success: default escape timeout")
sp.expect_prompt(
"\r\nsuccess: default escape timeout\r\n",
unmatched="vi replace line, default timeout: long delay",
)
# Test replacing a single character.
sp.send("echo TEXT")
sp.send("\033")
# Delay needed to allow fish to transition to vi "normal" mode.
sp.sleep(0.150)
sp.send("hhrAi\r")
sp.expect_prompt(
"\r\nTAXT\r\n", unmatched="vi mode replace char, default timeout: long delay"
)
# Test deleting characters with 'x'.
sp.send("echo MORE-TEXT")
sp.send("\033")
# Delay needed to allow fish to transition to vi "normal" mode.
sp.sleep(0.250)
sp.send("xxxxx\r")
# vi mode delete char, default timeout: long delay
sp.expect_prompt(
"\r\nMORE\r\n", unmatched="vi mode delete char, default timeout: long delay"
)
# Test jumping forward til before a character with t
sp.send("echo MORE-TEXT-IS-NICE")
sp.send("\033")
# Delay needed to allow fish to transition to vi "normal" mode.
sp.sleep(0.250)
sp.send("0tTD\r")
# vi mode forward-jump-till character, default timeout: long delay
sp.expect_prompt(
"\r\nMORE\r\n",
unmatched="vi mode forward-jump-till character, default timeout: long delay",
)
# Test jumping backward til before a character with T
sp.send("echo MORE-TEXT-IS-NICE")
sp.send("\033")
# Delay needed to allow fish to transition to vi "normal" mode.
sp.sleep(0.250)
sp.send("TSD\r")
# vi mode backward-jump-till character, default timeout: long delay
sp.expect_prompt(
"\r\nMORE-TEXT-IS\r\n",
unmatched="vi mode backward-jump-till character, default timeout: long delay",
)
# Test jumping backward with F and repeating
sp.send("echo MORE-TEXT-IS-NICE")
sp.send("\033")
# Delay needed to allow fish to transition to vi "normal" mode.
sp.sleep(0.250)
sp.send("F-;D\r")
# vi mode backward-jump-to character and repeat, default timeout: long delay
sp.expect_prompt(
"\r\nMORE-TEXT\r\n",
unmatched="vi mode backward-jump-to character and repeat, default timeout: long delay",
)
# Test jumping backward with F w/reverse jump
sp.send("echo MORE-TEXT-IS-NICE")
sp.send("\033")
# Delay needed to allow fish to transition to vi "normal" mode.
sp.sleep(0.250)
sp.send("F-F-,D\r")
# vi mode backward-jump-to character, and reverse, default timeout: long delay
sp.expect_prompt(
"\r\nMORE-TEXT-IS\r\n",
unmatched="vi mode backward-jump-to character, and reverse, default timeout: long delay",
)
# Verify that changing the escape timeout has an effect.
sp.send("set -g fish_escape_delay_ms 200\r")
sp.expect_prompt()
sp.send("echo fail: lengthened escape timeout")
sp.send("\033")
sp.sleep(0.350)
sp.send("ddi")
sp.send("echo success: lengthened escape timeout\r")
# vi replace line, 200ms timeout: long delay
sp.expect_prompt(
"\r\nsuccess: lengthened escape timeout\r\n",
unmatched="vi replace line, 200ms timeout: long delay",
)
# Verify that we don't switch to vi normal mode if we don't wait long enough
# after sending escape.
sp.send("echo fail: no normal mode")
sp.send("\033")
sp.sleep(0.050)
sp.send("ddi")
sp.send("inserted\r")
# vi replace line, 200ms timeout: short delay
sp.expect_prompt(
"\r\nfail: no normal modediinserted\r\n",
unmatched="vi replace line, 200ms timeout: short delay",
)
# Test 't' binding that contains non-zero arity function (forward-jump) followed
# by another function (and) https://github.com/fish-shell/fish-shell/issues/2357
sp.send("\033")
sp.sleep(0.300)
sp.send("ddiecho TEXT\033")
sp.sleep(0.300)
sp.send("hhtTrN\r")
sp.expect_prompt("\r\nTENT\r\n", unmatched="Couldn't find expected output 'TENT'")
# Test '~' (togglecase-char)
sp.send("\033")
sp.sleep(0.300)
sp.send("ccecho some TExT\033")
sp.sleep(0.300)
sp.send("hh~~bbve~\r")
sp.expect_prompt("\r\nSOME TeXT\r\n", unmatched="Couldn't find expected output 'SOME TeXT")
# Now test that exactly the expected bind modes are defined
sp.sendline("bind --list-modes")
sp.expect_prompt(
"\r\ndefault\r\ninsert\r\npaste\r\nreplace\r\nreplace_one\r\nvisual\r\n",
unmatched="Unexpected vi bind modes",
)
# Switch back to regular (emacs mode) key bindings.
sp.sendline("set -g fish_key_bindings fish_default_key_bindings")
sp.expect_prompt()
# Verify the custom escape timeout of 200ms set earlier is still in effect.
sp.sendline("echo fish_escape_delay_ms=$fish_escape_delay_ms")
sp.expect_prompt(
"\r\nfish_escape_delay_ms=200\r\n",
unmatched="default-mode custom timeout not set correctly",
)
# Set it to 100ms.
sp.sendline("set -g fish_escape_delay_ms 100")
sp.expect_prompt()
# Verify the emacs transpose word (\et) behavior using various delays,
# including none, after the escape character.
# Start by testing with no delay. This should transpose the words.
sp.send("echo abc def")
sp.send("\033")
sp.send("t\r")
# emacs transpose words, 100ms timeout: no delay
sp.expect_prompt(
"\r\ndef abc\r\n", unmatched="emacs transpose words fail, 100ms timeout: no delay"
)
# Same test as above but with a slight delay less than the escape timeout.
sp.send("echo ghi jkl")
sp.send("\033")
sp.sleep(0.080)
sp.send("t\r")
# emacs transpose words, 100ms timeout: short delay
sp.expect_prompt(
"\r\njkl ghi\r\n",
unmatched="emacs transpose words fail, 100ms timeout: short delay",
)
# Now test with a delay > the escape timeout. The transposition should not
# occur and the "t" should become part of the text that is echoed.
sp.send("echo mno pqr")
sp.send("\033")
sp.sleep(0.250)
sp.send("t\r")
# emacs transpose words, 100ms timeout: long delay
sp.expect_prompt(
"\r\nmno pqrt\r\n",
unmatched="emacs transpose words fail, 100ms timeout: long delay",
)
# Verify special characters, such as \cV, are not intercepted by the kernel
# tty driver. Rather, they can be bound and handled by fish.
sp.sendline("bind \\cV 'echo ctrl-v seen'")
sp.expect_prompt()
sp.send("\026\r")
sp.expect_prompt("ctrl-v seen", unmatched="ctrl-v not seen")
sp.send("bind \\cO 'echo ctrl-o seen'\r")
sp.expect_prompt()
sp.send("\017\r")
sp.expect_prompt("ctrl-o seen", unmatched="ctrl-o not seen")
# \x17 is ctrl-w.
sp.send("echo git@github.com:fish-shell/fish-shell")
sp.send("\x17\x17\r")
sp.expect_prompt("git@github.com:", unmatched="ctrl-w does not stop at :")
sp.send("echo git@github.com:fish-shell/fish-shell")
sp.send("\x17\x17\x17\r")
sp.expect_prompt("git@", unmatched="ctrl-w does not stop at @")
# Ensure that nul can be bound properly (#3189).
sp.send("bind -k nul 'echo nul seen'\r")
sp.expect_prompt
sp.send("\0" * 3)
sp.send("\r")
sp.expect_prompt("nul seen\r\nnul seen\r\nnul seen", unmatched="nul not seen")
# Test self-insert-notfirst. (#6603)
# Here the leading 'q's should be stripped, but the trailing ones not.
sp.sendline("bind q self-insert-notfirst")
sp.expect_prompt()
sp.sendline("qqqecho qqq")
sp.expect_prompt("qqq", unmatched="Leading qs not stripped")

28
tests/pexpects/pipeline.py Executable file
View file

@ -0,0 +1,28 @@
#!/usr/bin/env python3
from pexpect_helper import SpawnedProc
sp = SpawnedProc()
sp.expect_prompt()
sp.sendline("function echo_wrap ; /bin/echo $argv ; sleep 0.1; end")
sp.expect_prompt()
for i in range(5):
sp.sendline(
"echo_wrap 1 2 3 4 | $fish_test_helper become_foreground_then_print_stderr ; or exit 1"
)
sp.expect_prompt("become_foreground_then_print_stderr done")
# 'not' because we expect to have no jobs, in which case `jobs` will return false
sp.sendline("not jobs")
sp.expect_prompt("jobs: There are no jobs")
sp.sendline("function inner ; command true ; end; function outer; inner; end")
sp.expect_prompt()
for i in range(5):
sp.sendline(
"outer | $fish_test_helper become_foreground_then_print_stderr ; or exit 1"
)
sp.expect_prompt("become_foreground_then_print_stderr done")
sp.sendline("not jobs")
sp.expect_prompt("jobs: There are no jobs", unmatched="Should be no jobs")