From 3f129b570c6df47df7df871ce653c71b2fec424d Mon Sep 17 00:00:00 2001
From: Charles Ferguson <gerph@gerph.org>
Date: Thu, 29 Jun 2017 16:11:58 +0100
Subject: [PATCH] Add test harness for fish command invocation, and tests for
 init command.

The new '-C' initial command needs some tests, and as there are no
tests just yet for the command invocation, this change adds a harness
and calls it from the high-level tests in the Makefile.

The tests are similar in style to the other high level tests, in that
we capture the output and compare it to that which we expect. The
harness itself is written in bash - sorry - because we're testing the
fish shell's invocation, and trying to do that with the fish we've
just built wouldn't actually make for a very useful test when things
go wrong.

The 'tests/invocation.sh' script can be executed manually, or as part
of the 'make test' target, to make it easy to use both as part of the
development and as part of automation.

The harness has only been tested on linux with bash 4.3.11, and requires
grep and sed. Although not tested with OS X, I believe I have avoided
the syntax which is inconsistent.

The tests added here cover just the initial command's basic execution,
and when it is mixed with the regular '-c' command.
---
 Makefile.in                                   |  11 +-
 tests/invocation.sh                           | 242 ++++++++++++++++++
 tests/invocation/init-command-2.invoke        |   1 +
 tests/invocation/init-command-2.out           |   2 +
 .../init-command-mix-ordering.invoke          |   1 +
 .../invocation/init-command-mix-ordering.out  |   2 +
 tests/invocation/init-command-mix.invoke      |   1 +
 tests/invocation/init-command-mix.out         |   2 +
 tests/invocation/init-command.invoke          |   1 +
 tests/invocation/init-command.out             |   1 +
 10 files changed, 261 insertions(+), 3 deletions(-)
 create mode 100755 tests/invocation.sh
 create mode 100644 tests/invocation/init-command-2.invoke
 create mode 100644 tests/invocation/init-command-2.out
 create mode 100644 tests/invocation/init-command-mix-ordering.invoke
 create mode 100644 tests/invocation/init-command-mix-ordering.out
 create mode 100644 tests/invocation/init-command-mix.invoke
 create mode 100644 tests/invocation/init-command-mix.out
 create mode 100644 tests/invocation/init-command.invoke
 create mode 100644 tests/invocation/init-command.out

diff --git a/Makefile.in b/Makefile.in
index 7166c1887..2a64ad00c 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -350,14 +350,14 @@ test: test-prep install-force test_low_level test_high_level
 # We want the various tests to run serially so their output doesn't mix
 # We can do that by adding ordering dependencies based on what goals are being used.
 #
-test_goals := test_low_level test_fishscript test_interactive
+test_goals := test_low_level test_invocation test_fishscript test_interactive
 
 #
 # The following variables define targets that depend on the tests. If any more targets
 # are added that depend, directly or indirectly, on tests, they need to be recorded here.
 #
 test_test_deps = test_low_level $(test_high_level_test_deps)
-test_high_level_test_deps = test_fishscript test_interactive
+test_high_level_test_deps = test_invocation test_fishscript test_interactive
 
 active_test_goals = $(filter $(test_goals),$(foreach a,$(or $(MAKECMDGOALS),$(.DEFAULT_GOAL)),$(a) $($(a)_test_deps)))
 filter_up_to = $(eval b:=1)$(foreach a,$(2),$(and $(b),$(if $(subst $(1),,$(a)),$(a),$(eval b:=))))
@@ -371,9 +371,13 @@ test_low_level: fish_tests $(call filter_up_to,test_low_level,$(active_test_goal
 
 test_high_level: DESTDIR = $(PWD)/test/root/
 test_high_level: prefix = .
-test_high_level: test-prep install-force test_fishscript test_interactive
+test_high_level: test-prep install-force test_invocation test_fishscript test_interactive
 .PHONY: test_high_level
 
+test_invocation: $(call filter_up_to,test_invocation,$(active_test_goals))
+	cd tests; ./invocation.sh
+.PHONY: test_invocation
+
 test_fishscript: $(call filter_up_to,test_fishscript,$(active_test_goals))
 	cd tests; ../test/root/bin/fish test.fish
 .PHONY: test_fishscript
@@ -382,6 +386,7 @@ test_interactive: $(call filter_up_to,test_interactive,$(active_test_goals))
 	cd tests; ../test/root/bin/fish interactive.fish
 .PHONY: test_interactive
 
+
 #
 # commands.hdr collects documentation on all commands, functions and
 # builtins
diff --git a/tests/invocation.sh b/tests/invocation.sh
new file mode 100755
index 000000000..645031e97
--- /dev/null
+++ b/tests/invocation.sh
@@ -0,0 +1,242 @@
+#!/bin/bash
+##
+# Test that the invocation of the fish executable works as we hope.
+#
+# We try to run the 'fish' binary with different command line switches.
+# Each time we check against an output that we expect.
+#
+# We are testing fish's invocation itself, so this is not written in
+# fish itself - if the invocation wasn't working, we'd never even
+# be able to this test to check that the invocation wasn't working.
+#
+
+# Errors will be fatal
+set -e
+
+# If any command in the pipeline fails report the rc of the first fail.
+set -o pipefail
+
+# If nothing matches a glob expansion, return nothing (not the glob
+# itself)
+shopt -s nullglob
+
+# The directory this script is in (as everything is relative to here)
+here="$(cd "$(dirname "$0")" && pwd -P)"
+cd "$here"
+
+# The temporary directory to use
+temp_dir="$here/../test"
+
+# The files we're going to execute are in the 'invocation' directory.
+files_to_test=($(echo invocation/*.invoke))
+
+# The fish binary we are testing - for manual testing, may be overridden
+fish_exe="${fish_exe:-../test/root/bin/fish}"
+fish_dir="$(dirname "${fish_exe}")"
+fish_leaf="$(basename "${fish_exe}")"
+
+
+# Terminal colouring
+term_red="$(tput setaf 1)"
+term_green="$(tput setaf 2)"
+term_yellow="$(tput setaf 3)"
+term_blue="$(tput setaf 4)"
+term_magenta="$(tput setaf 5)"
+term_cyan="$(tput setaf 6)"
+term_white="$(tput setaf 7)"
+term_reset="$(tput sgr0)"
+
+
+##
+# Set variables to known values so that they will not affect the
+# execution of the test.
+function clean_environment() {
+
+    # Reset the terminal variables to a known type.
+    export TERM=xterm
+    unset ITERM_PROFILE
+
+    # And the language as well, so that we do not see differences in
+    # output dur to the user's locale
+    export LANGUAGE=en_US:en
+
+    # Ensure that the fish environment we use is in a clean state
+    rm -rf "${temp_dir}/data" "${temp_dir}/home"
+    mkdir -p "${temp_dir}/data" "${temp_dir}/home" "${temp_dir}/home/fish"
+    export XDG_DATA_HOME="${temp_dir}/data"
+    export XDG_CONFIG_HOME="${temp_dir}/home"
+}
+
+
+##
+# Fail completely :-(
+function fail() {
+    say red "FAIL: $@" >&2
+    exit 1
+}
+
+
+##
+# Coloured output
+function say() {
+    local color_name="$1"
+    local msg="$2"
+    local color_var="term_${color_name}"
+    local color="${!color_var}"
+
+    echo "$color$msg$term_reset"
+}
+
+
+##
+# Actual testing of a .invocation file.
+function test_file() {
+    local file="$*"
+    local dir="$(dirname "$file")"
+    local base="$(basename "$file" .invoke)"
+    local test_config="${dir}/${base}.config"
+    local test_stdout="${dir}/${base}.tmp.out"
+    local test_stderr="${dir}/${base}.tmp.err"
+    local want_stdout="${dir}/${base}.out"
+    local grep_stdout="${dir}/${base}.grep"
+    local want_stderr="${dir}/${base}.err"
+    local empty="${dir}/${base}.empty"
+    local -a filter
+    local rc=0
+    local test_args_literal
+    local test_args
+    local out_status=0
+    local err_status=0
+
+    # Literal arguments, for printing
+    test_args_literal="$(cat "$file")"
+    # Read the test arguments, escaping things that might be processed by us
+    test_args="$(sed 's/\$/\$/' "$file" | tr '\n' ' ')"
+
+    # Create an empty file so that we can compare against it if needed
+    echo -n > "${empty}"
+
+    # If they supplied a configuration file, we create it here
+    if [ -f "$test_config" ] ; then
+        cat "$test_config" > "${temp_dir}/home/fish/config.fish"
+    else
+        rm -f "${temp_dir}/home/fish/config.fish"
+    fi
+
+    # In some cases we want to check only a part of the output.
+    # For those we filter the output through grep'd matches.
+    if [ -f "$grep_stdout" ] ; then
+        # grep '-o', '-E' and '-f' are supported by the tools in modern GNU
+        # environments, and on OS X.
+        filter=('grep' '-o' '-E' '-f' "$grep_stdout")
+    else
+        filter=('cat')
+    fi
+
+    echo -n "Testing file $file ... "
+
+    # The hoops we are jumping through here, with changing directory are
+    # so that we always execute fish as './fish', which means that any
+    # error messages will appear the same, even if the tested binary
+    # is not one that we built here.
+    # We disable the exit-on-error here, so that we can catch the return
+    # code.
+    set +e
+    eval "cd \"$fish_dir\" && \"./$fish_leaf\" $test_args" \
+           2> "$test_stderr" \
+           < /dev/null       \
+           | ${filter[*]}    \
+           > "$test_stdout"
+    rc="$?"
+    set -e
+
+    if [ "$rc" != '0' ] ; then
+        # Write the return code on to the end of the stderr, so that it can be
+        # checked like anything else.
+        echo "RC: $rc" >> "${test_stderr}"
+    fi
+
+    # If the wanted output files are not present, they are assumed empty.
+    if [ ! -f "$want_stdout" ] ; then
+        want_stdout="$empty"
+    fi
+    if [ ! -f "$want_stderr" ] ; then
+        want_stderr="$empty"
+    fi
+
+    # The standard error that we get will report errors using non-relative
+    # filenames, so we try to replace these with the variable names.
+    #
+    # However, fish will also have helpfully translated the home directory
+    # into '~/' in the error report. Consequently, we need to perform a
+    # small fix-up so that we can replace the string sanely.
+    xdg_config_in_home="$XDG_CONFIG_HOME"
+    if [ "${xdg_config_in_home:0:${#HOME}}" = "${HOME}" ] ; then
+        xdg_config_in_home="~/${xdg_config_in_home:${#HOME}+1}"
+    fi
+    # 'sed -i' (inplace) has different syntax on BSD and GNU versions of
+    # the tool, so cannot be used here, hence we write to a separate file,
+    # and then move back.
+    sed "s,$xdg_config_in_home,\$XDG_CONFIG_HOME,g" "${test_stderr}" > "${test_stderr}.new"
+    mv -f "${test_stderr}.new" "${test_stderr}"
+
+    # Check the results
+    if ! diff "${test_stdout}" "${want_stdout}" >/dev/null 2>/dev/null ; then
+        out_status=1
+    fi
+    if ! diff "${test_stderr}" "${want_stderr}" >/dev/null 2>/dev/null ; then
+        err_status=1
+    fi
+
+    if [ "$out_status" = '0' ] && \
+       [ "$err_status" = '0' ] ; then
+        say green "ok"
+        # clean up tmp files
+        rm -f "${test_stdout}" "${test_stderr}" "${empty}"
+        rc=0
+    else
+        say red "fail"
+        say blue "$test_args_literal" | sed 's/^/    /'
+
+        if [ "$out_status" != '0' ] ; then
+            say yellow "Output differs for file $file. Diff follows:"
+            colordiff -u "${test_stdout}" "${want_stdout}"
+        fi
+        if [ "$err_status" != '0' ] ; then
+            say yellow "Error output differs for file $file. Diff follows:"
+            colordiff -u "${test_stderr}" "${want_stderr}"
+        fi
+        rc=1
+    fi
+
+    return $rc
+}
+
+
+########################################################################
+# Main harness
+
+if [ ! -x "${fish_exe}" ] ; then
+    fail "Fish executable not found at '${fish_exe}'"
+fi
+
+clean_environment
+
+say cyan "Testing shell invocation funtionality"
+
+passed=0
+failed=0
+for file in ${files_to_test[*]} ; do
+   if ! test_file "$file" ; then
+       failed=$(( failed + 1 ))
+   else
+       passed=$(( passed + 1 ))
+   fi
+done
+
+echo "Encountered $failed errors in the invocation tests (out of $(( failed + passed )))."
+
+if [ "$failed" != 0 ] ; then
+    exit 1
+fi
+exit 0
diff --git a/tests/invocation/init-command-2.invoke b/tests/invocation/init-command-2.invoke
new file mode 100644
index 000000000..667c6273f
--- /dev/null
+++ b/tests/invocation/init-command-2.invoke
@@ -0,0 +1 @@
+-C 'echo init-command' -C 'echo 2nd init-command'
diff --git a/tests/invocation/init-command-2.out b/tests/invocation/init-command-2.out
new file mode 100644
index 000000000..00962a99b
--- /dev/null
+++ b/tests/invocation/init-command-2.out
@@ -0,0 +1,2 @@
+init-command
+2nd init-command
diff --git a/tests/invocation/init-command-mix-ordering.invoke b/tests/invocation/init-command-mix-ordering.invoke
new file mode 100644
index 000000000..1718e5c03
--- /dev/null
+++ b/tests/invocation/init-command-mix-ordering.invoke
@@ -0,0 +1 @@
+-c 'echo command' -C 'echo init-command'
diff --git a/tests/invocation/init-command-mix-ordering.out b/tests/invocation/init-command-mix-ordering.out
new file mode 100644
index 000000000..8ca70f9a2
--- /dev/null
+++ b/tests/invocation/init-command-mix-ordering.out
@@ -0,0 +1,2 @@
+init-command
+command
diff --git a/tests/invocation/init-command-mix.invoke b/tests/invocation/init-command-mix.invoke
new file mode 100644
index 000000000..e40eb4088
--- /dev/null
+++ b/tests/invocation/init-command-mix.invoke
@@ -0,0 +1 @@
+-C 'echo init-command' -c 'echo command'
diff --git a/tests/invocation/init-command-mix.out b/tests/invocation/init-command-mix.out
new file mode 100644
index 000000000..8ca70f9a2
--- /dev/null
+++ b/tests/invocation/init-command-mix.out
@@ -0,0 +1,2 @@
+init-command
+command
diff --git a/tests/invocation/init-command.invoke b/tests/invocation/init-command.invoke
new file mode 100644
index 000000000..deb1f1061
--- /dev/null
+++ b/tests/invocation/init-command.invoke
@@ -0,0 +1 @@
+-C 'echo init-command'
diff --git a/tests/invocation/init-command.out b/tests/invocation/init-command.out
new file mode 100644
index 000000000..35db3ef38
--- /dev/null
+++ b/tests/invocation/init-command.out
@@ -0,0 +1 @@
+init-command