From 3fd1c3b64a23b761530bf26f74ba22fafe1bcf17 Mon Sep 17 00:00:00 2001
From: Paul Dreik <github@pauldreik.se>
Date: Mon, 11 Nov 2019 22:17:32 +0100
Subject: [PATCH] run short fuzzing and valgrind in github action
---
.github/workflows/fuzzers.yml | 104 +++++++++++++++++++++++
fuzz/build_fuzzer_variants.sh | 151 ++++++++++++++++++++++++++++++++++
2 files changed, 255 insertions(+)
create mode 100644 .github/workflows/fuzzers.yml
create mode 100755 fuzz/build_fuzzer_variants.sh
diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml
new file mode 100644
index 00000000..b93e4a13
--- /dev/null
+++ b/.github/workflows/fuzzers.yml
@@ -0,0 +1,104 @@
+name: Run fuzzers on stored corpus and test it with valgrind
+
+# In the case of a pull request happening at the same time as a cron
+# job, there is a risk two jobs run at the same time. Therefore,
+# the corpus is only uploaded for the master branch. Pull requests will
+# fuzz for a short while, but the results are not uploaded.
+on:
+ push:
+ pull_request:
+ schedule:
+ - cron: 23 */8 * * *
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ env:
+ allfuzzers: parser dump
+ steps:
+ - name: Install packages necessary for building
+ run: |
+ sudo apt-get install --quiet ninja-build valgrind
+ wget https://apt.llvm.org/llvm.sh
+ chmod +x llvm.sh
+ sudo ./llvm.sh 8
+
+ - uses: actions/checkout@v1
+ # with:
+ # remove this once integrated
+ # ref: paul/fuzz_experiment
+ - name: Download the corpus from the last run
+ run: |
+ wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar
+ tar xf corpus.tar
+ rm corpus.tar
+ - name: List clang versions
+ run: |
+ ls /usr/bin/clang*
+ which clang++
+ clang++ --version
+ - name: Build all the variants
+ run: fuzz/build_fuzzer_variants.sh
+ - name: Run the fastest fuzzer to explore fast
+ run: |
+ for fuzzer in $allfuzzers; do
+ mkdir -p out/$fuzzer # in case this is a new fuzzer, or corpus.tar is broken
+ build-ossfuzz-fast8/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=30
+ done
+ - name: Run the other fuzzer variants for $fuzzer, with sanitizers etc
+ run: |
+ for fuzzer in $allfuzzers; do
+ build-ossfuzz-withavx/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=20
+ build-ossfuzz-noavx/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=10
+ build-ossfuzz-noavx8/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=10
+ echo disable msan runs, it fails inside the fuzzing engine and not the fuzzed code!
+ echo build-ossfuzz-msan-noavx8/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=10 -reload=0
+ echo build-ossfuzz-msan-withavx8/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=10 -reload=0
+ echo now have $(ls out/$fuzzer |wc -l) files in corpus
+ done
+ - name: Minimize the corpus with the fast fuzzer
+ run: |
+ for fuzzer in $allfuzzers; do
+ mkdir -p out/cmin/$fuzzer
+ build-ossfuzz-fast8/fuzz/fuzz_$fuzzer -merge=1 out/cmin/$fuzzer out/$fuzzer
+ rm -rf out/$fuzzer
+ mv out/cmin/$fuzzer out/$fuzzer
+ done
+ - name: Package the corpus into an artifact
+ run: |
+ for fuzzer in $allfuzzers; do
+ tar rf corpus.tar out/$fuzzer
+ done
+ - name: Save the corpus as a github artifact
+ uses: actions/upload-artifact@v1
+ with:
+ name: corpus
+ path: corpus.tar
+ - name: Run the corpus through valgrind (normal build)
+ run: |
+ for fuzzer in $allfuzzers; do
+ find out/$fuzzer -type f |sort|xargs valgrind build-plain-noavx/fuzz/fuzz_$fuzzer 2>&1|tee valgrind-$fuzzer-noavx.txt
+ done
+ - name: Run the corpus through valgrind (noavx build)
+ run: |
+ for fuzzer in $allfuzzers; do
+ find out/$fuzzer -type f |sort|xargs valgrind build-plain-normal/fuzz/fuzz_$fuzzer 2>&1|tee valgrind-$fuzzer-normal.txt
+ done
+ - name: Compress the valgrind output
+ run: tar cf valgrind.tar valgrind-*.txt
+ - name: Save valgrind output as a github artifact
+ uses: actions/upload-artifact@v1
+ with:
+ name: valgrindresults
+ path: valgrind.tar
+ - name: Upload the corpus and results to bintray if we are on master
+ run: |
+ if [ $(git rev-parse --verify HEAD) = $(git rev-parse --verify origin/master) ] ; then
+ echo uploading each artifact twice, otherwise it will not be published
+ curl -T corpus.tar -upauldreik:${{ secrets.bintrayApiKey }} https://api.bintray.com/content/pauldreik/simdjson-fuzz-corpus/corpus/0/corpus/corpus.tar";publish=1;override=1"
+ curl -T corpus.tar -upauldreik:${{ secrets.bintrayApiKey }} https://api.bintray.com/content/pauldreik/simdjson-fuzz-corpus/corpus/0/corpus/corpus.tar";publish=1;override=1"
+ curl -T valgrind.tar -upauldreik:${{ secrets.bintrayApiKey }} https://api.bintray.com/content/pauldreik/simdjson-fuzz-corpus/corpus/0/corpus/valgrind.tar";publish=1;override=1"
+ curl -T valgrind.tar -upauldreik:${{ secrets.bintrayApiKey }} https://api.bintray.com/content/pauldreik/simdjson-fuzz-corpus/corpus/0/corpus/valgrind.tar";publish=1;override=1"
+ else
+ echo "not on master, won't upload to bintray"
+ fi
diff --git a/fuzz/build_fuzzer_variants.sh b/fuzz/build_fuzzer_variants.sh
new file mode 100755
index 00000000..59c42203
--- /dev/null
+++ b/fuzz/build_fuzzer_variants.sh
@@ -0,0 +1,151 @@
+#!/bin/sh
+#
+# This file builds multiple variants of the fuzzers
+# - different sanitizers
+# - different build options
+# - reproduce build, for running through valgrind
+
+# fail on error
+set -eu
+
+unset CXX CC CFLAGS CXXFLAGS LDFLAGS
+
+# A reproduce build, without avx but otherwise as plain
+# as it gets. No sanitizers or optimization.
+variant=plain-noavx
+if [ ! -d build-$variant ] ; then
+ mkdir build-$variant
+ cd build-$variant
+
+ cmake .. \
+ -GNinja \
+ -DCMAKE_BUILD_TYPE=Debug \
+ -DSIMDJSON_BUILD_STATIC=On \
+ -DENABLE_FUZZING=On \
+ -DSIMDJSON_FUZZ_LINKMAIN=On \
+ -DSIMDJSON_DISABLE_AVX=On
+
+ ninja
+ cd ..
+fi
+
+# A reproduce build as plain as it gets. Everythings tunable is
+# using the defaults.
+variant=plain-normal
+if [ ! -d build-$variant ] ; then
+ mkdir build-$variant
+ cd build-$variant
+
+ cmake .. \
+ -GNinja \
+ -DCMAKE_BUILD_TYPE=Debug \
+ -DSIMDJSON_BUILD_STATIC=On \
+ -DENABLE_FUZZING=On \
+ -DSIMDJSON_FUZZ_LINKMAIN=On
+
+ ninja
+ cd ..
+fi
+
+# a fuzzer with sanitizers, built with avx disabled.
+variant=ossfuzz-noavx
+if [ ! -d build-$variant ] ; then
+
+ export CC=clang
+ export CXX="clang++"
+ export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined -mno-avx2 -mno-avx "
+ export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined -mno-avx2 -mno-avx"
+ export LIB_FUZZING_ENGINE="-fsanitize=fuzzer"
+
+ mkdir build-$variant
+ cd build-$variant
+
+ cmake .. \
+ -GNinja \
+ -DCMAKE_BUILD_TYPE=Debug \
+ -DSIMDJSON_BUILD_STATIC=On \
+ -DENABLE_FUZZING=On \
+ -DSIMDJSON_FUZZ_LINKMAIN=Off \
+ -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE \
+ -DSIMDJSON_DISABLE_AVX=On
+
+ ninja
+ cd ..
+fi
+
+
+# a fuzzer with sanitizers, built with avx disabled.
+variant=ossfuzz-noavx8
+if [ ! -d build-$variant ] ; then
+
+ export CC=clang-8
+ export CXX="clang++-8"
+ export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined -mno-avx2 -mno-avx "
+ export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined -mno-avx2 -mno-avx"
+ export LIB_FUZZING_ENGINE="-fsanitize=fuzzer"
+
+ mkdir build-$variant
+ cd build-$variant
+
+ cmake .. \
+ -GNinja \
+ -DCMAKE_BUILD_TYPE=Debug \
+ -DSIMDJSON_BUILD_STATIC=On \
+ -DENABLE_FUZZING=On \
+ -DSIMDJSON_FUZZ_LINKMAIN=Off \
+ -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE \
+ -DSIMDJSON_DISABLE_AVX=On
+
+ ninja
+ cd ..
+fi
+
+# a fuzzer with sanitizers, default built
+variant=ossfuzz-withavx
+if [ ! -d build-$variant ] ; then
+
+ export CC=clang
+ export CXX="clang++"
+ export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined"
+ export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined"
+ export LIB_FUZZING_ENGINE="-fsanitize=fuzzer"
+
+ mkdir build-$variant
+ cd build-$variant
+
+ cmake .. \
+ -GNinja \
+ -DCMAKE_BUILD_TYPE=Debug \
+ -DSIMDJSON_BUILD_STATIC=On \
+ -DENABLE_FUZZING=On \
+ -DSIMDJSON_FUZZ_LINKMAIN=Off \
+ -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE
+
+ ninja
+ cd ..
+fi
+
+# a fast fuzzer, for fast exploration
+variant=ossfuzz-fast8
+if [ ! -d build-$variant ] ; then
+ export CC=clang-8
+ export CXX="clang++-8"
+ export CFLAGS="-fsanitize=fuzzer-no-link -O3 -g"
+ export CXXFLAGS="-fsanitize=fuzzer-no-link -O3 -g"
+ export LIB_FUZZING_ENGINE="-fsanitize=fuzzer"
+
+ mkdir build-$variant
+ cd build-$variant
+
+ cmake .. \
+ -GNinja \
+ -DCMAKE_BUILD_TYPE= \
+ -DSIMDJSON_BUILD_STATIC=On \
+ -DENABLE_FUZZING=On \
+ -DSIMDJSON_FUZZ_LINKMAIN=Off \
+ -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE
+
+ ninja
+
+ cd ..
+fi