Merge pull request #119 from StevenVanAcker/better-testing

Better travis-ci testing, including for fedora and archlinux
This commit is contained in:
Yan 2017-02-21 22:46:17 -08:00 committed by GitHub
commit 80a19b7f92
6 changed files with 132 additions and 177 deletions

View file

@ -3,186 +3,80 @@ sudo: required
services:
- docker
before_install:
- if [ "$DISTRO" = "archlinux" ]; then docker build -t ctftools -f Dockerfile.archlinux .; fi
- if [ "$DISTRO" = "fedora" ]; then docker build -t ctftools -f Dockerfile.fedora .; fi
- if [ "$DISTRO" = "xenial" ]; then docker build -t ctftools -f Dockerfile.xenial .; fi
- if [ "$DISTRO" = "trusty" ]; then docker build -t ctftools -f Dockerfile .; fi
env:
- DISTRO="trusty" TOOL="afl"
- DISTRO="trusty" TOOL="android-sdk"
- DISTRO="trusty" TOOL="angr" EXPECTFAIL=1
- DISTRO="trusty" TOOL="apktool"
- DISTRO="trusty" TOOL="barf"
- DISTRO="trusty" TOOL="beef" EXPECTFAIL=1 ### unsupported for now, needs ruby with RVM
- DISTRO="trusty" TOOL="bindead"
- DISTRO="trusty" TOOL="binwalk"
- DISTRO="trusty" TOOL="burpsuite"
- DISTRO="trusty" TOOL="capstone"
- DISTRO="trusty" TOOL="checksec"
- DISTRO="trusty" TOOL="codereason"
- DISTRO="trusty" TOOL="commix"
- DISTRO="trusty" TOOL="cribdrag"
#- DISTRO="trusty" TOOL="cross2" ### takes too long for travis
#- DISTRO="trusty" TOOL="crosstool" ### takes too long for travis
- DISTRO="trusty" TOOL="df"
- DISTRO="trusty" TOOL="dirb"
- DISTRO="trusty" TOOL="dirsearch"
- DISTRO="trusty" TOOL="dislocker"
- DISTRO="trusty" TOOL="elfkickers"
- DISTRO="trusty" TOOL="elfparser"
- DISTRO="trusty" TOOL="evilize"
- DISTRO="trusty" TOOL="exetractor"
- DISTRO="trusty" TOOL="featherduster"
- DISTRO="trusty" TOOL="firmware-mod-kit"
- DISTRO="trusty" TOOL="foresight"
- DISTRO="trusty" TOOL="gdb"
- DISTRO="trusty" TOOL="gdb-heap"
- DISTRO="trusty" TOOL="gef"
- DISTRO="trusty" TOOL="hash-identifier"
- DISTRO="trusty" TOOL="hashkill"
- DISTRO="trusty" TOOL="hashpump"
- DISTRO="trusty" TOOL="hashpump-partialhash"
- DISTRO="trusty" TOOL="honggfuzz"
- DISTRO="trusty" TOOL="jdgui"
- DISTRO="trusty" TOOL="keystone"
- DISTRO="trusty" TOOL="libheap"
- DISTRO="trusty" TOOL="littleblackbox"
- DISTRO="trusty" TOOL="mitmproxy"
- DISTRO="trusty" TOOL="msieve"
- DISTRO="trusty" TOOL="panda" EXPECTFAIL=1 ### not sure why this fails, compilation errors
- DISTRO="trusty" TOOL="pathgrind" EXPECTFAIL=1 ### only works on kernel 2.4 or 2.6
- DISTRO="trusty" TOOL="pdf-parser"
- DISTRO="trusty" TOOL="peda"
- DISTRO="trusty" TOOL="peepdf"
- DISTRO="trusty" TOOL="pemcrack"
- DISTRO="trusty" TOOL="pkcrack"
- DISTRO="trusty" TOOL="preeny"
- DISTRO="trusty" TOOL="pwndbg"
- DISTRO="trusty" TOOL="pwntools" EXPECTFAIL=1
- DISTRO="trusty" TOOL="python-paddingoracle"
- DISTRO="trusty" TOOL="python-pin"
- DISTRO="trusty" TOOL="pyvmmonitor"
# Tests for fedora
- DISTRO="fedora" TOOL="apktool binwalk burpsuite capstone checksec commix
cribdrag dirsearch elfkickers evilize exetractor foresight gdb gdb-heap gef
jdgui libheap pdf-parser peda peepdf pkcrack python-paddingoracle pyvmmonitor
radare2 reveng ropper rp++ scrdec18 shellnoob shellsploit ssh_decoder
steganabara stegsolve subbrute unicorn virtualsocket"
- DISTRO="fedora" TOOL="xortool yafu z3"
- DISTRO="fedora" EXPECTFAIL=1 TOOL="afl android-sdk angr barf beef bindead
codereason df dirb dislocker elfparser featherduster firmware-mod-kit
hash-identifier hashkill hashpump hashpump-partialhash honggfuzz keystone
littleblackbox mitmproxy msieve panda pathgrind pemcrack preeny pwndbg pwntools
python-pin qemu qira snowman sonic-visualizer sqlmap sslsplit stegdetect
taintgrind testdisk tor-browser veles villoc wcc xrop xspy zsteg"
# - DISTRO="fedora" TOOL="cross2 crosstool" ### takes too long for travis-ci
# Tests for archlinux
- DISTRO="archlinux" TOOL="afl android-sdk apktool binwalk burpsuite checksec
commix cribdrag df dirb dirsearch evilize exetractor featherduster
firmware-mod-kit foresight gdb gdb-heap gef"
- DISTRO="archlinux" TOOL="hashpump hashpump-partialhash jdgui keystone libheap
msieve pdf-parser peda peepdf pemcrack pkcrack pwndbg pwntools
python-paddingoracle pyvmmonitor radare2 reveng rp++ scrdec18 shellnoob
ssh_decoder sslsplit steganabara stegsolve subbrute testdisk tor-browser
unicorn veles virtualsocket xortool xrop yafu zsteg elfkickers honggfuzz"
- DISTRO="archlinux" EXPECTFAIL=1 TOOL="angr barf beef bindead capstone
codereason dislocker elfparser hash-identifier hashkill littleblackbox
mitmproxy panda pathgrind preeny qira ropper shellsploit"
- DISTRO="archlinux" EXPECTFAIL=1 TOOL="snowman sonic-visualizer sqlmap
stegdetect taintgrind villoc wcc xspy z3 python-pin"
# - DISTRO="archlinux" TOOL="cross2 crosstool qemu" ### takes too long for travis-ci
# Tests for trusty, grouped together to most optimally use a single build process
- DISTRO="trusty" TOOL="afl android-sdk apktool barf binwalk burpsuite capstone
checksec codereason"
- DISTRO="trusty" TOOL="commix cribdrag df dirb dirsearch dislocker elfkickers
elfparser evilize exetractor jdgui keystone libheap littleblackbox mitmproxy
msieve pdf-parser peda peepdf pemcrack scrdec18 shellnoob shellsploit snowman
sqlmap ssh_decoder sslsplit steganabara stegsolve subbrute bindead hashkill
python-pin qira stegdetect villoc zsteg"
- DISTRO="trusty" TOOL="featherduster firmware-mod-kit foresight gdb gdb-heap
gef hash-identifier hashpump hashpump-partialhash honggfuzz testdisk
tor-browser unicorn virtualsocket xortool xspy yafu z3"
- DISTRO="trusty" TOOL="pkcrack preeny pwndbg python-paddingoracle pyvmmonitor
radare2 reveng ropper rp++"
- DISTRO="trusty" TOOL="qemu"
- DISTRO="trusty" TOOL="qira"
- DISTRO="trusty" TOOL="radare2"
- DISTRO="trusty" TOOL="reveng"
- DISTRO="trusty" TOOL="ropper"
- DISTRO="trusty" TOOL="rp++"
- DISTRO="trusty" TOOL="scrdec18"
- DISTRO="trusty" TOOL="shellnoob"
- DISTRO="trusty" TOOL="shellsploit"
- DISTRO="trusty" TOOL="snowman"
- DISTRO="trusty" TOOL="sonic-visualizer" EXPECTFAIL=1 ### build errors
- DISTRO="trusty" TOOL="sqlmap"
- DISTRO="trusty" TOOL="ssh_decoder"
- DISTRO="trusty" TOOL="sslsplit"
- DISTRO="trusty" TOOL="steganabara"
- DISTRO="trusty" TOOL="stegdetect"
- DISTRO="trusty" TOOL="stegsolve"
- DISTRO="trusty" TOOL="subbrute"
- DISTRO="trusty" TOOL="taintgrind" EXPECTFAIL=1 ### missing Iop_* declarations
- DISTRO="trusty" TOOL="testdisk"
- DISTRO="trusty" TOOL="tor-browser"
- DISTRO="trusty" TOOL="unicorn"
- DISTRO="trusty" TOOL="veles" EXPECTFAIL=1 ### Cmake is too old
- DISTRO="trusty" TOOL="villoc"
- DISTRO="trusty" TOOL="virtualsocket"
- DISTRO="trusty" TOOL="wcc" EXPECTFAIL=1 ### depends on xenial
- DISTRO="trusty" TOOL="xortool"
- DISTRO="trusty" TOOL="xrop" EXPECTFAIL=1
- DISTRO="trusty" TOOL="xspy"
- DISTRO="trusty" TOOL="yafu"
- DISTRO="trusty" TOOL="z3"
- DISTRO="trusty" TOOL="zsteg"
- DISTRO="trusty" EXPECTFAIL=1 TOOL="angr beef panda pathgrind pwntools
sonic-visualizer taintgrind veles wcc xrop"
# - DISTRO="trusty" TOOL="cross2 crosstool" ### takes too long for travis-ci
- DISTRO="xenial" TOOL="afl"
- DISTRO="xenial" TOOL="android-sdk"
- DISTRO="xenial" TOOL="angr"
- DISTRO="xenial" TOOL="apktool"
- DISTRO="xenial" TOOL="barf"
- DISTRO="xenial" TOOL="beef" EXPECTFAIL=1 ### unsupported for now, needs ruby with RVM
- DISTRO="xenial" TOOL="bindead" EXPECTFAIL=1
- DISTRO="xenial" TOOL="binwalk"
- DISTRO="xenial" TOOL="burpsuite"
- DISTRO="xenial" TOOL="capstone"
- DISTRO="xenial" TOOL="checksec"
- DISTRO="xenial" TOOL="codereason"
- DISTRO="xenial" TOOL="commix"
- DISTRO="xenial" TOOL="cribdrag"
#- DISTRO="xenial" TOOL="cross2" ### takes too long for travis
#- DISTRO="xenial" TOOL="crosstool" ### takes too long for travis
- DISTRO="xenial" TOOL="df"
- DISTRO="xenial" TOOL="dirb"
- DISTRO="xenial" TOOL="dirsearch"
- DISTRO="xenial" TOOL="dislocker"
- DISTRO="xenial" TOOL="elfkickers"
- DISTRO="xenial" TOOL="elfparser"
- DISTRO="xenial" TOOL="evilize"
- DISTRO="xenial" TOOL="exetractor"
- DISTRO="xenial" TOOL="featherduster"
- DISTRO="xenial" TOOL="firmware-mod-kit"
- DISTRO="xenial" TOOL="foresight"
- DISTRO="xenial" TOOL="gdb"
- DISTRO="xenial" TOOL="gdb-heap"
- DISTRO="xenial" TOOL="gef"
- DISTRO="xenial" TOOL="hash-identifier"
- DISTRO="xenial" TOOL="hashkill" EXPECTFAIL=1
- DISTRO="xenial" TOOL="hashpump"
- DISTRO="xenial" TOOL="hashpump-partialhash"
- DISTRO="xenial" TOOL="honggfuzz"
- DISTRO="xenial" TOOL="jdgui"
- DISTRO="xenial" TOOL="keystone"
- DISTRO="xenial" TOOL="libheap"
- DISTRO="xenial" TOOL="littleblackbox"
- DISTRO="xenial" TOOL="mitmproxy"
- DISTRO="xenial" TOOL="msieve"
- DISTRO="xenial" TOOL="panda" EXPECTFAIL=1 ### not sure why this fails, compilation errors
- DISTRO="xenial" TOOL="pathgrind" EXPECTFAIL=1 ### only works on kernel 2.4 or 2.6
- DISTRO="xenial" TOOL="pdf-parser"
- DISTRO="xenial" TOOL="peda"
- DISTRO="xenial" TOOL="peepdf"
- DISTRO="xenial" TOOL="pemcrack"
- DISTRO="xenial" TOOL="pkcrack"
- DISTRO="xenial" TOOL="preeny"
- DISTRO="xenial" TOOL="pwndbg"
- DISTRO="xenial" TOOL="pwntools" EXPECTFAIL=1
- DISTRO="xenial" TOOL="python-paddingoracle"
- DISTRO="xenial" TOOL="python-pin" EXPECTFAIL=1
- DISTRO="xenial" TOOL="pyvmmonitor"
# Tests for xenial, grouped together to most optimally use a single build process
- DISTRO="xenial" TOOL="afl android-sdk angr apktool barf binwalk burpsuite
capstone checksec codereason"
- DISTRO="xenial" TOOL="commix cribdrag df dirb dirsearch dislocker elfkickers
elfparser evilize exetractor jdgui keystone libheap littleblackbox mitmproxy
msieve pdf-parser peda peepdf pemcrack scrdec18 shellnoob shellsploit snowman
sqlmap ssh_decoder sslsplit steganabara stegsolve subbrute"
- DISTRO="xenial" TOOL="featherduster firmware-mod-kit foresight gdb gdb-heap
gef hash-identifier hashpump hashpump-partialhash honggfuzz testdisk
tor-browser unicorn veles virtualsocket xortool xspy yafu z3"
- DISTRO="xenial" TOOL="pkcrack preeny pwndbg python-paddingoracle pyvmmonitor
radare2 reveng ropper rp++"
- DISTRO="xenial" TOOL="qemu"
- DISTRO="xenial" TOOL="qira" EXPECTFAIL=1
- DISTRO="xenial" TOOL="radare2"
- DISTRO="xenial" TOOL="reveng"
- DISTRO="xenial" TOOL="ropper"
- DISTRO="xenial" TOOL="rp++"
- DISTRO="xenial" TOOL="scrdec18"
- DISTRO="xenial" TOOL="shellnoob"
- DISTRO="xenial" TOOL="shellsploit"
- DISTRO="xenial" TOOL="snowman"
- DISTRO="xenial" TOOL="sonic-visualizer" EXPECTFAIL=1 ### build errors
- DISTRO="xenial" TOOL="sqlmap"
- DISTRO="xenial" TOOL="ssh_decoder"
- DISTRO="xenial" TOOL="sslsplit"
- DISTRO="xenial" TOOL="steganabara"
- DISTRO="xenial" TOOL="stegdetect" EXPECTFAIL=1
- DISTRO="xenial" TOOL="stegsolve"
- DISTRO="xenial" TOOL="subbrute"
- DISTRO="xenial" TOOL="taintgrind" EXPECTFAIL=1 ### missing Iop_* declarations
- DISTRO="xenial" TOOL="testdisk"
- DISTRO="xenial" TOOL="tor-browser"
- DISTRO="xenial" TOOL="unicorn"
- DISTRO="xenial" TOOL="veles"
- DISTRO="xenial" TOOL="villoc" EXPECTFAIL=1
- DISTRO="xenial" TOOL="virtualsocket"
- DISTRO="xenial" TOOL="wcc" EXPECTFAIL=1 ### depends on xenial
- DISTRO="xenial" TOOL="xortool"
- DISTRO="xenial" TOOL="xrop" EXPECTFAIL=1
- DISTRO="xenial" TOOL="xspy"
- DISTRO="xenial" TOOL="yafu"
- DISTRO="xenial" TOOL="z3"
- DISTRO="xenial" TOOL="zsteg" EXPECTFAIL=1
- DISTRO="xenial" EXPECTFAIL=1 TOOL="beef bindead hashkill panda pathgrind
pwntools python-pin qira sonic-visualizer stegdetect taintgrind villoc wcc xrop
zsteg"
# - DISTRO="xenial" TOOL="cross2 crosstool" ### takes too long for travis-ci
script:
- docker run --rm ctftools lsb_release -a
- docker run -e EXPECTFAIL="$EXPECTFAIL" -e TOOL="$TOOL" --rm ctftools bash -ic 'manage-tools -s -f -v test $TOOL'
- ./bin/travis-ci.sh

19
Dockerfile.fedora Normal file
View file

@ -0,0 +1,19 @@
FROM fedora
RUN dnf -y install which sudo git redhat-lsb
RUN useradd -m ctf
COPY .git /home/ctf/tools/.git
RUN chown -R ctf.ctf /home/ctf/tools
RUN echo "ctf ALL=NOPASSWD: ALL" > /etc/sudoers.d/ctf
USER ctf
WORKDIR /home/ctf/tools
RUN git checkout .
RUN bin/manage-tools -s setup
RUN bin/ctf-tools-pip install appdirs
RUN echo "workon ctftools" >> /home/ctf/.bashrc
WORKDIR /home/ctf
CMD bash -i

View file

@ -2,7 +2,7 @@
echo "Need to get ruby with RVM... Unsupported for now"
exit 1
apt-get install build-essential openssl libreadline6 \
apt-get -y install build-essential openssl libreadline6 \
libreadline6-dev zlib1g zlib1g-dev libssl-dev \
libyaml-dev libsqlite3-0 libsqlite3-dev sqlite3 \
libxml2-dev libxslt1-dev autoconf libc6-dev \

42
bin/travis-ci.sh Executable file
View file

@ -0,0 +1,42 @@
#!/bin/bash -e
export EXPECTFAIL=${EXPECTFAIL:-0}
starttime=$SECONDS
failed=""
for t in $TOOL;
do
set +e
toolstarttime=$SECONDS
echo "[-] TOOL $t TEST STARTED: $((SECONDS - starttime)) seconds since start of script."
if ! docker run -e EXPECTFAIL="$EXPECTFAIL" -e TOOL="$t" --rm ctftools bash -ic 'manage-tools -s -f -v test $TOOL';
then
failed="$failed$t "
fi
echo "[-] TOOL $t TEST ENDED: $((SECONDS - toolstarttime)) seconds, $((SECONDS - starttime)) seconds since start of script."
set -e
done
if [ "$failed" != "" ];
then
echo "==================================================="
failcount=$(echo "$failed" | wc -w)
totalcount=$(echo "$TOOL" | wc -w)
if [ "$EXPECTFAIL" -eq "1" ];
then
echo "ERROR: $failcount/$totalcount tools succeeded while they were expected to fail: $failed"
else
echo "ERROR: $failcount/$totalcount tools failed while they should have succeeded: $failed"
fi
echo "==================================================="
exit 1
fi
if [ "$EXPECTFAIL" -eq "1" ];
then
echo "DONE: $totalcount tools failed as expected."
else
echo "DONE: $totalcount tools succeeded as expected."
fi
exit 0

View file

@ -1,5 +1,5 @@
#!/bin/bash -ex
set -eu -o pipefail
apt-get install libbfd-dev libunwind8-dev
# apt-get install libbfd-dev libunwind8-dev
pacman -Syu --noconfirm --needed libunwind binutils

View file

@ -1,7 +1,7 @@
#!/bin/bash -ex
set -eu -o pipefail
apt-get install clang libbfd-dev uthash-dev \
apt-get -y install clang libbfd-dev uthash-dev \
libelf-dev libcapstone-dev \
libreadline6 libreadline6-dev \
libiberty-dev libgsl-dev