diff --git a/cmake/Tests.cmake b/cmake/Tests.cmake
index 7b398744e..7c78ab6f4 100644
--- a/cmake/Tests.cmake
+++ b/cmake/Tests.cmake
@@ -26,6 +26,9 @@ IF(NOT FISH_IN_TREE_BUILD)
     ADD_DEPENDENCIES(fish_tests tests_dir)
 ENDIF()
 
+# Copy littlecheck.py
+CONFIGURE_FILE(build_tools/littlecheck.py littlecheck.py COPYONLY)
+
 # Create the 'test' target.
 # Set a policy so CMake stops complaining about the name 'test'.
 CMAKE_POLICY(PUSH)
diff --git a/tests/printf.in b/tests/checks/printf.fish
similarity index 72%
rename from tests/printf.in
rename to tests/checks/printf.fish
index d7174253e..16677f5ce 100644
--- a/tests/printf.in
+++ b/tests/checks/printf.fish
@@ -1,31 +1,53 @@
+# RUN: %fish %s
+
 printf "Hello %d %i %f %F %g %G\n" 1 2 3 4 5 6
+# CHECK: Hello 1 2 3.000000 4.000000 5 6
 
 printf "%x %X %o %llu\n" 10 11 8 -1
+# CHECK: a B 10 18446744073709551615
+
 # %a has OS-dependent output - see #1139
 #printf "%a %A\n" 14 15
 
 printf "%c %s\n" a hello
+# CHECK: a hello
+
 printf "%c%c%c\n" hello … o
+# CHECK: h…o
+
 printf "%e %E\n" 5 6
+# CHECK: 5.000000e+00 6.000000E+00
 
 printf "%20d\n" 50
+# CHECK:                   50
+
 printf "%-20d%d\n" 5 10
+# CHECK: 5                   10
 
 printf "%*d\n" 10 100
+# CHECK:       100
 
 printf "%%\"\\\n"
 printf "%s\b%s\n" x y
-printf "abc\rdef\n"
-printf "Msg1\fMsg2\n"
-printf "foo\vbar\vbaz\n"
-printf "\111 \x50 \u0051 \U00000052"
+# CHECK: %"\nxy
 
+printf "abc\rdef\n"
+# CHECK: abc{{\r}}def
+
+printf "Msg1\fMsg2\n"
+# CHECK: Msg1{{\f}}Msg2
+
+printf "foo\vbar\vbaz\n"
+# CHECK: foobarbaz
+
+printf "\111 \x50 \u0051 \U00000052"
 echo
-echo "Test escapes"
+# CHECK: I P Q R
 
 # \c escape means "stop printing"
 printf 'a\cb'
 echo
+# CHECK: a
 
 # Bogus printf specifier, should produce no stdout
 printf "%5" 10 2>/dev/null
@@ -33,12 +55,18 @@ printf "%5" 10 2>/dev/null
 # Octal escapes produce literal bytes, not characters
 # \376 is 0xFE
 printf '\376' | display_bytes
+# CHECK: 0000000 376
+# CHECK: 0000001
 
 # Verify that floating point conversions and output work correctly with
 # different combinations of locales and floating point strings. See issue
 # #3334. This starts by assuming an locale using english conventions.
 printf '%e\n' "1.23"  # should succeed, output should be 1.230000e+00
+# CHECK: 1.230000e+00
+
 printf '%e\n' "2,34"  # should fail
+# CHECK: 2.000000e+00
+# CHECKERR: 2,34: value not completely converted
 
 # Try to use one of several locales that use a comma as the decimal mark
 # rather than the period used in english speaking locales. If we don't find
@@ -63,16 +91,28 @@ else
     echo '3,450000e+00'
     echo '4,560000e+00'
 end
+# CHECK: 3,450000e+00
+# CHECK: 4,560000e+00
 
 # Verify long long ints are handled correctly. See issue #3352.
 printf 'long hex1 %x\n' 498216206234
+# CHECK: long hex1 73ffffff9a
 printf 'long hex2 %X\n' 498216206234
+# CHECK: long hex2 73FFFFFF9A
 printf 'long hex3 %X\n' 0xABCDEF1234567890
+# CHECK: long hex3 ABCDEF1234567890
 printf 'long hex4 %X\n' 0xABCDEF12345678901
+# CHECKERR: 0xABCDEF12345678901: Number out of range
 printf 'long decimal %d\n' 498216206594
+# CHECK: long hex4 long decimal 498216206594
 printf 'long signed %d\n' -498216206595
+# CHECK: long signed -498216206595
 printf 'long signed to unsigned %u\n' -498216206596
+# CHECK: long signed to unsigned 18446743575493345020
 
 # Verify numeric conversion still happens even if it couldn't be fully converted
 printf '%d\n' 15.1
+# CHECK: 15
+# CHECKERR: 15.1: value not completely converted
 echo $status
+# CHECK: 1
diff --git a/tests/printf.err b/tests/printf.err
deleted file mode 100644
index d8d4de233..000000000
--- a/tests/printf.err
+++ /dev/null
@@ -1,3 +0,0 @@
-2,34: value not completely converted
-0xABCDEF12345678901: Number out of range
-15.1: value not completely converted
diff --git a/tests/printf.out b/tests/printf.out
deleted file mode 100644
index 40f19f791..000000000
--- a/tests/printf.out
+++ /dev/null
@@ -1,29 +0,0 @@
-Hello 1 2 3.000000 4.000000 5 6
-a B 10 18446744073709551615
-a hello
-h…o
-5.000000e+00 6.000000E+00
-                  50
-5                   10
-       100
-%"\nxy
-abc
def
-Msg1Msg2
-foobarbaz
-I P Q R
-Test escapes
-a
-0000000 376
-0000001
-1.230000e+00
-2.000000e+00
-3,450000e+00
-4,560000e+00
-long hex1 73ffffff9a
-long hex2 73FFFFFF9A
-long hex3 ABCDEF1234567890
-long hex4 long decimal 498216206594
-long signed -498216206595
-long signed to unsigned 18446743575493345020
-15
-1
diff --git a/tests/test.fish b/tests/test.fish
index 62025fe0d..62e929794 100644
--- a/tests/test.fish
+++ b/tests/test.fish
@@ -15,7 +15,7 @@ cd (dirname (status -f))
 if set -q argv[1]
     set files_to_test $argv.in
 else
-    set files_to_test *.in
+    set files_to_test *.in checks/*.fish
 end
 
 # These env vars should not be inherited from the user environment because they can affect the
@@ -29,7 +29,7 @@ or exit
 
 say -o cyan "Testing high level script functionality"
 
-function test_file
+function test_in_file
     set -l file $argv[1]
     set -l base (basename $file .in)
 
@@ -69,10 +69,24 @@ function test_file
     end
 end
 
+function test_littlecheck_file
+    set -l file $argv[1]
+    echo "Testing file $file"
+    ../littlecheck.py -s fish=../test/root/bin/fish $file
+end
+
 set -l failed
 for i in $files_to_test
-    if not test_file $i
-        set failed $failed $i
+    if string match --quiet '*.fish' $i
+        if not test_littlecheck_file $i
+            # littlecheck test
+            set failed $failed $i
+        end
+    else
+        # .in test
+        if not test_in_file $i
+            set failed $failed $i
+        end
     end
 end