From 3fd1c3b64a23b761530bf26f74ba22fafe1bcf17 Mon Sep 17 00:00:00 2001 From: Paul Dreik Date: Mon, 11 Nov 2019 22:17:32 +0100 Subject: [PATCH] run short fuzzing and valgrind in github action --- .github/workflows/fuzzers.yml | 104 +++++++++++++++++++++++ fuzz/build_fuzzer_variants.sh | 151 ++++++++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 .github/workflows/fuzzers.yml create mode 100755 fuzz/build_fuzzer_variants.sh diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml new file mode 100644 index 00000000..b93e4a13 --- /dev/null +++ b/.github/workflows/fuzzers.yml @@ -0,0 +1,104 @@ +name: Run fuzzers on stored corpus and test it with valgrind + +# In the case of a pull request happening at the same time as a cron +# job, there is a risk two jobs run at the same time. Therefore, +# the corpus is only uploaded for the master branch. Pull requests will +# fuzz for a short while, but the results are not uploaded. +on: + push: + pull_request: + schedule: + - cron: 23 */8 * * * + +jobs: + build: + runs-on: ubuntu-latest + env: + allfuzzers: parser dump + steps: + - name: Install packages necessary for building + run: | + sudo apt-get install --quiet ninja-build valgrind + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 8 + + - uses: actions/checkout@v1 + # with: + # remove this once integrated + # ref: paul/fuzz_experiment + - name: Download the corpus from the last run + run: | + wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar + tar xf corpus.tar + rm corpus.tar + - name: List clang versions + run: | + ls /usr/bin/clang* + which clang++ + clang++ --version + - name: Build all the variants + run: fuzz/build_fuzzer_variants.sh + - name: Run the fastest fuzzer to explore fast + run: | + for fuzzer in $allfuzzers; do + mkdir -p out/$fuzzer # in case this is a new fuzzer, or corpus.tar is broken + build-ossfuzz-fast8/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=30 + done + - name: Run the other fuzzer variants for $fuzzer, with sanitizers etc + run: | + for fuzzer in $allfuzzers; do + build-ossfuzz-withavx/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=20 + build-ossfuzz-noavx/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=10 + build-ossfuzz-noavx8/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=10 + echo disable msan runs, it fails inside the fuzzing engine and not the fuzzed code! + echo build-ossfuzz-msan-noavx8/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=10 -reload=0 + echo build-ossfuzz-msan-withavx8/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=10 -reload=0 + echo now have $(ls out/$fuzzer |wc -l) files in corpus + done + - name: Minimize the corpus with the fast fuzzer + run: | + for fuzzer in $allfuzzers; do + mkdir -p out/cmin/$fuzzer + build-ossfuzz-fast8/fuzz/fuzz_$fuzzer -merge=1 out/cmin/$fuzzer out/$fuzzer + rm -rf out/$fuzzer + mv out/cmin/$fuzzer out/$fuzzer + done + - name: Package the corpus into an artifact + run: | + for fuzzer in $allfuzzers; do + tar rf corpus.tar out/$fuzzer + done + - name: Save the corpus as a github artifact + uses: actions/upload-artifact@v1 + with: + name: corpus + path: corpus.tar + - name: Run the corpus through valgrind (normal build) + run: | + for fuzzer in $allfuzzers; do + find out/$fuzzer -type f |sort|xargs valgrind build-plain-noavx/fuzz/fuzz_$fuzzer 2>&1|tee valgrind-$fuzzer-noavx.txt + done + - name: Run the corpus through valgrind (noavx build) + run: | + for fuzzer in $allfuzzers; do + find out/$fuzzer -type f |sort|xargs valgrind build-plain-normal/fuzz/fuzz_$fuzzer 2>&1|tee valgrind-$fuzzer-normal.txt + done + - name: Compress the valgrind output + run: tar cf valgrind.tar valgrind-*.txt + - name: Save valgrind output as a github artifact + uses: actions/upload-artifact@v1 + with: + name: valgrindresults + path: valgrind.tar + - name: Upload the corpus and results to bintray if we are on master + run: | + if [ $(git rev-parse --verify HEAD) = $(git rev-parse --verify origin/master) ] ; then + echo uploading each artifact twice, otherwise it will not be published + curl -T corpus.tar -upauldreik:${{ secrets.bintrayApiKey }} https://api.bintray.com/content/pauldreik/simdjson-fuzz-corpus/corpus/0/corpus/corpus.tar";publish=1;override=1" + curl -T corpus.tar -upauldreik:${{ secrets.bintrayApiKey }} https://api.bintray.com/content/pauldreik/simdjson-fuzz-corpus/corpus/0/corpus/corpus.tar";publish=1;override=1" + curl -T valgrind.tar -upauldreik:${{ secrets.bintrayApiKey }} https://api.bintray.com/content/pauldreik/simdjson-fuzz-corpus/corpus/0/corpus/valgrind.tar";publish=1;override=1" + curl -T valgrind.tar -upauldreik:${{ secrets.bintrayApiKey }} https://api.bintray.com/content/pauldreik/simdjson-fuzz-corpus/corpus/0/corpus/valgrind.tar";publish=1;override=1" + else + echo "not on master, won't upload to bintray" + fi diff --git a/fuzz/build_fuzzer_variants.sh b/fuzz/build_fuzzer_variants.sh new file mode 100755 index 00000000..59c42203 --- /dev/null +++ b/fuzz/build_fuzzer_variants.sh @@ -0,0 +1,151 @@ +#!/bin/sh +# +# This file builds multiple variants of the fuzzers +# - different sanitizers +# - different build options +# - reproduce build, for running through valgrind + +# fail on error +set -eu + +unset CXX CC CFLAGS CXXFLAGS LDFLAGS + +# A reproduce build, without avx but otherwise as plain +# as it gets. No sanitizers or optimization. +variant=plain-noavx +if [ ! -d build-$variant ] ; then + mkdir build-$variant + cd build-$variant + + cmake .. \ + -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DSIMDJSON_BUILD_STATIC=On \ + -DENABLE_FUZZING=On \ + -DSIMDJSON_FUZZ_LINKMAIN=On \ + -DSIMDJSON_DISABLE_AVX=On + + ninja + cd .. +fi + +# A reproduce build as plain as it gets. Everythings tunable is +# using the defaults. +variant=plain-normal +if [ ! -d build-$variant ] ; then + mkdir build-$variant + cd build-$variant + + cmake .. \ + -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DSIMDJSON_BUILD_STATIC=On \ + -DENABLE_FUZZING=On \ + -DSIMDJSON_FUZZ_LINKMAIN=On + + ninja + cd .. +fi + +# a fuzzer with sanitizers, built with avx disabled. +variant=ossfuzz-noavx +if [ ! -d build-$variant ] ; then + + export CC=clang + export CXX="clang++" + export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined -mno-avx2 -mno-avx " + export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined -mno-avx2 -mno-avx" + export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" + + mkdir build-$variant + cd build-$variant + + cmake .. \ + -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DSIMDJSON_BUILD_STATIC=On \ + -DENABLE_FUZZING=On \ + -DSIMDJSON_FUZZ_LINKMAIN=Off \ + -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE \ + -DSIMDJSON_DISABLE_AVX=On + + ninja + cd .. +fi + + +# a fuzzer with sanitizers, built with avx disabled. +variant=ossfuzz-noavx8 +if [ ! -d build-$variant ] ; then + + export CC=clang-8 + export CXX="clang++-8" + export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined -mno-avx2 -mno-avx " + export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined -mno-avx2 -mno-avx" + export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" + + mkdir build-$variant + cd build-$variant + + cmake .. \ + -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DSIMDJSON_BUILD_STATIC=On \ + -DENABLE_FUZZING=On \ + -DSIMDJSON_FUZZ_LINKMAIN=Off \ + -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE \ + -DSIMDJSON_DISABLE_AVX=On + + ninja + cd .. +fi + +# a fuzzer with sanitizers, default built +variant=ossfuzz-withavx +if [ ! -d build-$variant ] ; then + + export CC=clang + export CXX="clang++" + export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined" + export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined" + export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" + + mkdir build-$variant + cd build-$variant + + cmake .. \ + -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DSIMDJSON_BUILD_STATIC=On \ + -DENABLE_FUZZING=On \ + -DSIMDJSON_FUZZ_LINKMAIN=Off \ + -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE + + ninja + cd .. +fi + +# a fast fuzzer, for fast exploration +variant=ossfuzz-fast8 +if [ ! -d build-$variant ] ; then + export CC=clang-8 + export CXX="clang++-8" + export CFLAGS="-fsanitize=fuzzer-no-link -O3 -g" + export CXXFLAGS="-fsanitize=fuzzer-no-link -O3 -g" + export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" + + mkdir build-$variant + cd build-$variant + + cmake .. \ + -GNinja \ + -DCMAKE_BUILD_TYPE= \ + -DSIMDJSON_BUILD_STATIC=On \ + -DENABLE_FUZZING=On \ + -DSIMDJSON_FUZZ_LINKMAIN=Off \ + -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE + + ninja + + cd .. +fi